diff --git a/packages/gittensory-miner/.gittensory-miner.env.example b/packages/gittensory-miner/.gittensory-miner.env.example new file mode 100644 index 0000000000..a00534f6e0 --- /dev/null +++ b/packages/gittensory-miner/.gittensory-miner.env.example @@ -0,0 +1,11 @@ +# Env file for docker-compose.miner.yml (#5177). Copy to `.gittensory-miner.env`, fill in real values, and do +# NOT commit the filled-in copy. Values are intentionally EMPTY here (a real-looking value in a committed file +# trips secret scanners). Compose reads `KEY=value` lines; leave a key blank or delete it if unused. +# +# Required: a GitHub token the miner uses to read issues and open PRs. +GITHUB_TOKEN= +# Optional: only for the coding-agent providers you enable (claude-cli / codex-cli / agent-sdk). +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +# Optional: override the on-container state dir (defaults to /data/miner, mapped to the miner-data volume). +GITTENSORY_MINER_CONFIG_DIR=/data/miner diff --git a/packages/gittensory-miner/DEPLOYMENT.md b/packages/gittensory-miner/DEPLOYMENT.md index d0a5f06f0d..4e38dffa5c 100644 --- a/packages/gittensory-miner/DEPLOYMENT.md +++ b/packages/gittensory-miner/DEPLOYMENT.md @@ -2,13 +2,13 @@ Two form factors for running `@jsonbored/gittensory-miner`: **laptop mode** (single machine, zero Docker) and **fleet mode** (containerized workers with a shared data volume). Both are 100% client-side for core operation — the miner never uploads source and never requires a hosted Gittensory callback to boot. Credentials (GitHub tokens, etc.) stay on the operator's machine or in their own secret store; nothing is baked into images. -| | Laptop mode | Fleet mode | -|---|---|---| -| **Best for** | One contributor machine, local experimentation | Many parallel miner attempts on a host or small cluster | -| **Dependencies** | Node.js `>=22.13.0` only | Docker (or compatible runtime) + Node image or custom image | -| **State** | SQLite files under `~/.config/gittensory-miner/` (override with `GITTENSORY_MINER_CONFIG_DIR`) | Same SQLite layout on a mounted `/data` (or `GITTENSORY_MINER_CONFIG_DIR`) volume | -| **Setup** | `npm install -g @jsonbored/gittensory-miner` or workspace build | `docker build` + `docker run` with env + volume (see below) | -| **Footprint** | One Node process, local disk for ledgers/queues | One container per worker; scale horizontally by adding containers | +| | Laptop mode | Fleet mode | +| ---------------- | ---------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| **Best for** | One contributor machine, local experimentation | Many parallel miner attempts on a host or small cluster | +| **Dependencies** | Node.js `>=22.13.0` only | Docker (or compatible runtime) + Node image or custom image | +| **State** | SQLite files under `~/.config/gittensory-miner/` (override with `GITTENSORY_MINER_CONFIG_DIR`) | Same SQLite layout on a mounted `/data` (or `GITTENSORY_MINER_CONFIG_DIR`) volume | +| **Setup** | `npm install -g @jsonbored/gittensory-miner` or workspace build | `docker build` + `docker run` with env + volume (see below) | +| **Footprint** | One Node process, local disk for ledgers/queues | One container per worker; scale horizontally by adding containers | ## Laptop mode walkthrough @@ -69,6 +69,17 @@ The image entrypoint is `gittensory-miner`; pass subcommands after the image nam The repo-root [`docker-compose.yml`](../../docker-compose.yml) documents the **self-hosted review stack** (the `gittensory` API/orb), not the miner CLI. Miners are clients of that stack (or of github.com directly) and do not require it to run locally. +### Docker Compose (fleet mode) + +Instead of a hand-assembled `docker run`, [`docker-compose.miner.yml`](docker-compose.miner.yml) defines a long-lived `miner` service (built from this package's Dockerfile, `restart: unless-stopped`, state on a named `miner-data` volume). Credentials come from an env file, never inlined: + +```sh +cp .gittensory-miner.env.example .gittensory-miner.env # fill in GITHUB_TOKEN (+ optional provider keys) +docker compose -f docker-compose.miner.yml up -d --build +``` + +**Scaling to N parallel workers.** `docker compose -f docker-compose.miner.yml up -d --scale miner=N` gives every replica the **same** `miner-data` volume — and the miner's SQLite ledgers are **not** safe for concurrent access, so N replicas on one volume will contend/corrupt. To run N **isolated** workers, give each its own state: run N separate compose projects (`docker compose -p miner-1 …`, `-p miner-2 …` — `-p` namespaces the volume) or point each at a distinct `GITTENSORY_MINER_CONFIG_DIR` on its own mount. For built-in isolated horizontal scaling, use the Kubernetes StatefulSet in [`k8s/`](../../k8s/) (per-pod volumes). + ## Invariants - Core miner bookkeeping (claims, plans, queues, ledgers) works offline after install. diff --git a/packages/gittensory-miner/docker-compose.miner.yml b/packages/gittensory-miner/docker-compose.miner.yml new file mode 100644 index 0000000000..33bf3489fe --- /dev/null +++ b/packages/gittensory-miner/docker-compose.miner.yml @@ -0,0 +1,40 @@ +# docker-compose for gittensory-miner (AMS) fleet mode (#5177). +# +# Runs the miner as a long-lived CLI worker instead of a hand-assembled `docker run`. Build context is the +# MONOREPO ROOT (../..) because the Dockerfile needs the full workspace on disk before `npm ci` — see the +# comments in packages/gittensory-miner/Dockerfile. +# +# Usage: +# 1. Create the env file (never commit real values): cp .gittensory-miner.env.example .gittensory-miner.env +# 2. Start one worker: docker compose -f docker-compose.miner.yml up -d --build +# +# Scaling to N parallel workers: +# docker compose -f docker-compose.miner.yml up -d --scale miner=N +# BUT: `--scale` gives every replica the SAME `miner-data` volume, and the miner's SQLite ledgers +# (claim-ledger.sqlite3, plan-store.sqlite3, …) are NOT safe for concurrent multi-process access — N replicas +# sharing one volume WILL corrupt/contend on those files. To run N ISOLATED workers, give each its own state: +# • run N separate compose projects, each with its own volume: +# docker compose -p miner-1 -f docker-compose.miner.yml up -d +# docker compose -p miner-2 -f docker-compose.miner.yml up -d # …etc; -p namespaces the volume +# • or point each worker at a distinct GITTENSORY_MINER_CONFIG_DIR on its own mount. +# (A single-shared-volume `--scale` is only safe for one active worker; Kubernetes StatefulSet per-pod PVCs, see +# k8s/, are the built-in answer for isolated horizontal scaling.) +services: + miner: + build: + context: ../.. + dockerfile: packages/gittensory-miner/Dockerfile + image: gittensory-miner:latest + # ENTRYPOINT is `gittensory-miner`; `run` is the continuous fleet-worker loop. + command: ["run"] + restart: unless-stopped + # Operator-supplied secrets/config (GITHUB_TOKEN, optional provider keys). Copy the .example, fill it in, + # and NEVER commit the real file — env vars stay out of the compose file and out of `docker inspect` args. + env_file: + - .gittensory-miner.env + environment: + GITTENSORY_MINER_CONFIG_DIR: /data/miner + volumes: + - miner-data:/data/miner +volumes: + miner-data: diff --git a/test/unit/miner-docker-compose.test.ts b/test/unit/miner-docker-compose.test.ts new file mode 100644 index 0000000000..940e4226c9 --- /dev/null +++ b/test/unit/miner-docker-compose.test.ts @@ -0,0 +1,50 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +// Validation for the AMS fleet-mode compose file (#5177). Not `src/**` logic, so Codecov doesn't gate it — but +// the structural contract (service built from the package Dockerfile, named volume for SQLite persistence, +// restart policy, credentials via env-file not inlined) is asserted here as a real test, and the env example is +// checked to ship only empty (secret-scanner-safe) placeholders. +const MINER_DIR = join(process.cwd(), "packages/gittensory-miner"); +const compose = parse( + readFileSync(join(MINER_DIR, "docker-compose.miner.yml"), "utf8"), +) as Record; +const envExample = readFileSync( + join(MINER_DIR, ".gittensory-miner.env.example"), + "utf8", +); + +describe("docker-compose.miner.yml (#5177)", () => { + it("defines a miner service built from the package Dockerfile (monorepo-root context)", () => { + const miner = compose.services?.miner; + expect(miner).toBeTruthy(); + expect(miner.build.dockerfile).toBe("packages/gittensory-miner/Dockerfile"); + expect(miner.build.context).toBe("../.."); + expect(miner.command).toContain("run"); + }); + + it("persists state on a named volume and restarts unless stopped", () => { + const miner = compose.services.miner; + expect(miner.restart).toBe("unless-stopped"); + expect(miner.volumes).toContain("miner-data:/data/miner"); + expect(compose.volumes).toHaveProperty("miner-data"); + expect(miner.environment.GITTENSORY_MINER_CONFIG_DIR).toBe("/data/miner"); + }); + + it("sources credentials from an env file, never hardcoded in the compose", () => { + const miner = compose.services.miner; + expect(miner.env_file).toContain(".gittensory-miner.env"); + // no secret-named key is assigned a value directly in the compose `environment` block + expect(JSON.stringify(miner.environment ?? {})).not.toMatch( + /TOKEN|API_KEY|SECRET|PASSWORD/i, + ); + }); + + it("ships an env example with empty (scanner-safe) placeholder values for every credential", () => { + for (const key of ["GITHUB_TOKEN", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"]) { + expect(envExample).toMatch(new RegExp(`^${key}=\\s*$`, "m")); + } + }); +});