diff --git a/.env.example b/.env.example index 5f914452bb..19adaf83bd 100644 --- a/.env.example +++ b/.env.example @@ -368,11 +368,30 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # LITESTREAM_ACCESS_KEY_ID= # LITESTREAM_SECRET_ACCESS_KEY= # LITESTREAM_ENDPOINT= # e.g. s3.us-west-002.backblazeb2.com (omit for AWS S3) +# # With --profile storage, point at local MinIO instead: +# # LITESTREAM_ENDPOINT=http://minio:9000 +# # LITESTREAM_ACCESS_KEY_ID=${MINIO_ROOT_USER} +# # LITESTREAM_SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD} # LITESTREAM_REGION=us-east-1 # BACKUP_ACKNOWLEDGED=false # silences the boot-time warning about running SQLite with no # acknowledged backup once you've wired Litestream (or an # equivalent) above. Default false (warning shown). +# --- n8n workflow automation (--profile workflows) --- +# N8N_PASSWORD=changeme # REQUIRED at runtime when using --profile workflows +# N8N_USER=admin +# N8N_WEBHOOK_URL=https://n8n.example.com # set to public URL if receiving external webhooks +# N8N_ENCRYPTION_KEY= # 32-char random string; persists credentials across restarts +# N8N_MEM_LIMIT=512m + +# --- MinIO S3-compatible object storage (--profile storage) --- +# MINIO_ROOT_USER=minio # REQUIRED when using --profile storage +# MINIO_ROOT_PASSWORD=changeme # REQUIRED when using --profile storage +# MINIO_BUCKET=gittensory # default bucket name (create via console at http://localhost:9001) +# MINIO_MEM_LIMIT=1g +# Console at http://localhost:9001 +# S3 API at http://minio:9000 (from other containers) or http://localhost:9000 (from host) + # --- Queue worker (#977/#1201) --- # QUEUE_CONCURRENCY=4 # max concurrent job-processing loops per instance. Default 4; set 1 # # for strict serial processing. 8 is a reasonable starting point for diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index b0cef4fb7d..b58fe16a8b 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -92,6 +92,11 @@ jobs: - name: Validate self-host source map run: node scripts/validate-selfhost-sourcemap.mjs + - name: Validate docker-compose.yml + run: | + docker compose config --quiet + docker compose --profile workflows --profile storage config --quiet + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 - name: Build the Docker image @@ -107,6 +112,11 @@ jobs: docker run --rm --entrypoint sh gittensory:selfhost-ci -c \ 'command -v claude && claude --version && command -v codex && codex --version' + - name: Free disk before visual-review image build + run: | + docker image prune -f + docker builder prune -f --filter until=24h || true + - name: Build release target with visual review deps run: | docker buildx build \ @@ -117,6 +127,8 @@ jobs: -t gittensory:selfhost-prebuilt-visual-ci . docker run --rm --entrypoint node gittensory:selfhost-prebuilt-visual-ci \ -e "import('puppeteer-core').then(() => console.log('puppeteer-core ok'))" + docker rmi gittensory:selfhost-prebuilt-visual-ci || true + docker image prune -f - name: Boot the container + smoke-test /health, /ready, /metrics, migrations run: | diff --git a/docker-compose.yml b/docker-compose.yml index 5dcac30a98..6ec55ea7fe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,8 @@ # --profile visual-review Headless Chromium (browserless) for before/after PR screenshot capture # --profile rees Review-enrichment service (REES) for heavier PR analysis, in-network only # --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream +# --profile workflows n8n workflow automation (Slack/Teams/email fan-out, scheduled reports) +# --profile storage MinIO S3-compatible object store (Litestream destination, artifact blobs) # --profile caddy Caddy HTTPS terminator with auto-TLS (set DOMAIN= in .env) # --profile observability Prometheus + Alertmanager + Loki + Promtail + Grafana (pre-wired) # --profile tailscale Tailscale sidecar — access the stack via your tailnet @@ -31,6 +33,8 @@ # docker compose --profile postgres --profile caddy up -d # Postgres + HTTPS # docker compose --profile observability up -d # metrics + logs + dashboards # docker compose --profile rees up -d # local REES review enrichment +# docker compose --profile workflows up -d # n8n automation at :5678 +# docker compose --profile storage up -d # MinIO S3 API :9000, console :9001 # docker compose --profile tailscale --profile runners up -d # tailnet + CI runners # # HOST CLOCK (#3811): containers share the HOST kernel clock, so this compose file cannot fix a @@ -660,6 +664,73 @@ services: LITESTREAM_ENDPOINT: ${LITESTREAM_ENDPOINT:-} LITESTREAM_REGION: ${LITESTREAM_REGION:-us-east-1} + # ── n8n workflow automation (--profile workflows) ───────────────────────── + # Optional no-code automation: webhook fan-out (Slack/Teams/email), scheduled gate summaries, + # GitHub issue auto-triage from review signals. Import bundled templates from n8n/workflows/ + # via http://localhost:5678/workflows/import. Set N8N_WEBHOOK_URL to your public URL when + # receiving webhooks from outside the host. Zero impact on the review engine when inactive. + n8n: + image: n8nio/n8n:2.31.0 + restart: unless-stopped + <<: *default-logging + profiles: ["workflows"] + ports: + - "5678:5678" + volumes: + - n8n-data:/home/node/.n8n + environment: + N8N_BASIC_AUTH_ACTIVE: "true" + N8N_BASIC_AUTH_USER: ${N8N_USER:-admin} + # Soft default, not ":?required" -- compose interpolates every service's env vars for the whole file + # even when --profile workflows isn't active, so a hard-required var here would break `docker compose up` + # for every self-hoster not using this profile (same reasoning as BROWSERLESS_TOKEN / REES_SHARED_SECRET). + # The entrypoint below fails closed at container start when the profile is active but N8N_PASSWORD is unset. + N8N_BASIC_AUTH_PASSWORD: ${N8N_PASSWORD:-} + WEBHOOK_URL: ${N8N_WEBHOOK_URL:-http://localhost:5678} + N8N_ENCRYPTION_KEY: ${N8N_ENCRYPTION_KEY:-} + GENERIC_TIMEZONE: ${TZ:-UTC} + deploy: + resources: + limits: + memory: "${N8N_MEM_LIMIT:-512m}" + entrypoint: + - /bin/sh + - -ec + - | + if [ -z "$${N8N_BASIC_AUTH_PASSWORD:-}" ]; then + echo >&2 "Set N8N_PASSWORD in .env before using --profile workflows." + exit 1 + fi + exec /docker-entrypoint.sh + + # ── MinIO S3-compatible object storage (--profile storage) ────────────── + # Local blob store: Litestream backup destination (set LITESTREAM_ENDPOINT=http://minio:9000), + # visual-review screenshot artifacts, Orb event archive. Console UI at http://localhost:9001. + # S3 API: http://minio:9000 from other containers, http://localhost:9000 from the host. + minio: + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + restart: unless-stopped + <<: *default-logging + profiles: ["storage"] + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio-data:/data + environment: + # Soft defaults for the same whole-file interpolation reason as N8N_PASSWORD above. MinIO itself + # refuses to start without credentials when this profile is active -- fail closed, not open. + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-} + command: server /data --console-address ":9001" + deploy: + resources: + limits: + memory: "${MINIO_MEM_LIMIT:-1g}" + # No healthcheck: minio/minio is scratch/distroless (no shell, curl, wget, or mc), so an + # in-container probe cannot run -- same constraint as grafana/loki above. Readiness is observable + # at GET /minio/health/live on :9000 once the server binds. + # ── Caddy (--profile caddy) ──────────────────────────────────────────────── # Auto-TLS via Let's Encrypt. Set DOMAIN=reviews.yourcompany.com in .env, then # remove the plain `ports:` entry from the loopover service above. @@ -1213,6 +1284,8 @@ volumes: runner-work: loopover-backups: tempo-data: + n8n-data: + minio-data: networks: # The implicit project network every service joins by default (declared so it can be referenced diff --git a/n8n/workflows/gate-daily-summary.json b/n8n/workflows/gate-daily-summary.json new file mode 100644 index 0000000000..72c795135f --- /dev/null +++ b/n8n/workflows/gate-daily-summary.json @@ -0,0 +1,78 @@ +{ + "name": "Gittensory Gate Daily Summary", + "nodes": [ + { + "parameters": { + "rule": { + "interval": [ + { + "field": "cronExpression", + "expression": "0 9 * * *" + } + ] + } + }, + "id": "schedule-daily", + "name": "Daily 09:00", + "type": "n8n-nodes-base.scheduleTrigger", + "typeVersion": 1.2, + "position": [0, 0] + }, + { + "parameters": { + "method": "GET", + "url": "={{ $env.GITTENSORY_GATE_SUMMARY_URL ?? 'http://loopover:8787/v1/app/selfhost/gate/summary' }}", + "options": {} + }, + "id": "fetch-summary", + "name": "Fetch gate summary", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [280, 0] + }, + { + "parameters": { + "method": "POST", + "url": "={{ $env.GITTENSORY_SUMMARY_WEBHOOK_URL ?? 'https://example.com/webhook' }}", + "sendBody": true, + "specifyBody": "json", + "jsonBody": "={{ $json }}", + "options": {} + }, + "id": "post-summary", + "name": "Post summary webhook", + "type": "n8n-nodes-base.httpRequest", + "typeVersion": 4.2, + "position": [560, 0] + } + ], + "connections": { + "Daily 09:00": { + "main": [ + [ + { + "node": "Fetch gate summary", + "type": "main", + "index": 0 + } + ] + ] + }, + "Fetch gate summary": { + "main": [ + [ + { + "node": "Post summary webhook", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {}, + "meta": { + "templateCredsSetupCompleted": false, + "instanceId": "gittensory-gate-daily-summary" + } +} diff --git a/n8n/workflows/issue-auto-triage.json b/n8n/workflows/issue-auto-triage.json new file mode 100644 index 0000000000..7677470029 --- /dev/null +++ b/n8n/workflows/issue-auto-triage.json @@ -0,0 +1,97 @@ +{ + "name": "Gittensory Issue Auto-Triage", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "gittensory-pr-closed", + "responseMode": "onReceived", + "options": {} + }, + "id": "webhook-pr-closed", + "name": "PR auto-closed webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 0], + "webhookId": "gittensory-pr-closed" + }, + { + "parameters": { + "conditions": { + "options": { + "caseSensitive": true, + "leftValue": "", + "typeValidation": "strict" + }, + "conditions": [ + { + "id": "has-linked-issue", + "leftValue": "={{ $json.body.linkedIssueNumber ?? $json.linkedIssueNumber }}", + "rightValue": "", + "operator": { + "type": "number", + "operation": "exists" + } + } + ], + "combinator": "and" + }, + "options": {} + }, + "id": "filter-linked-issue", + "name": "Has linked issue?", + "type": "n8n-nodes-base.if", + "typeVersion": 2, + "position": [280, 0] + }, + { + "parameters": { + "owner": "={{ $json.body.owner ?? $json.owner }}", + "repository": "={{ $json.body.repo ?? $json.repo }}", + "issueNumber": "={{ $json.body.linkedIssueNumber ?? $json.linkedIssueNumber }}", + "body": "=PR #{{ $json.body.prNumber ?? $json.prNumber }} was auto-closed by the loopover gate.\n\nReason: {{ $json.body.closeReason ?? $json.closeReason ?? 'see gate comment on the PR' }}" + }, + "id": "github-comment", + "name": "Comment on linked issue", + "type": "n8n-nodes-base.github", + "typeVersion": 1.1, + "position": [560, 0], + "credentials": { + "githubApi": { + "id": "CONFIGURE_GITHUB_CREDENTIAL", + "name": "GitHub account" + } + } + } + ], + "connections": { + "PR auto-closed webhook": { + "main": [ + [ + { + "node": "Has linked issue?", + "type": "main", + "index": 0 + } + ] + ] + }, + "Has linked issue?": { + "main": [ + [ + { + "node": "Comment on linked issue", + "type": "main", + "index": 0 + } + ], + [] + ] + } + }, + "pinData": {}, + "meta": { + "templateCredsSetupCompleted": false, + "instanceId": "gittensory-issue-auto-triage" + } +} diff --git a/n8n/workflows/review-slack-notify.json b/n8n/workflows/review-slack-notify.json new file mode 100644 index 0000000000..eb076a3d58 --- /dev/null +++ b/n8n/workflows/review-slack-notify.json @@ -0,0 +1,55 @@ +{ + "name": "Gittensory Review → Slack", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "gittensory-review", + "responseMode": "onReceived", + "options": {} + }, + "id": "webhook-review", + "name": "Gittensory review webhook", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [0, 0], + "webhookId": "gittensory-review" + }, + { + "parameters": { + "channel": "#reviews", + "text": "=Review {{ $json.body.verdict ?? $json.verdict ?? 'unknown' }} for {{ $json.body.repo ?? $json.repo ?? 'repo' }} PR #{{ $json.body.prNumber ?? $json.prNumber ?? '?' }}", + "otherOptions": {} + }, + "id": "slack-notify", + "name": "Post to Slack", + "type": "n8n-nodes-base.slack", + "typeVersion": 2.2, + "position": [320, 0], + "credentials": { + "slackApi": { + "id": "CONFIGURE_SLACK_CREDENTIAL", + "name": "Slack account" + } + } + } + ], + "connections": { + "Gittensory review webhook": { + "main": [ + [ + { + "node": "Post to Slack", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "pinData": {}, + "meta": { + "templateCredsSetupCompleted": false, + "instanceId": "gittensory-review-slack-notify" + } +} diff --git a/test/unit/selfhost-compose-workflows-storage.test.ts b/test/unit/selfhost-compose-workflows-storage.test.ts new file mode 100644 index 0000000000..8732575948 --- /dev/null +++ b/test/unit/selfhost-compose-workflows-storage.test.ts @@ -0,0 +1,173 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseDocument } from "yaml"; +import { describe, expect, it } from "vitest"; + +function readYamlWithMerge(path: string): Record { + const doc = parseDocument(readFileSync(path, "utf8"), { merge: true }); + expect(doc.errors, `YAML parse errors in ${path}`).toHaveLength(0); + const value = doc.toJS(); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${path} must be a YAML object`); + } + return value as Record; +} + +function record(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error("expected object"); + } + return value as Record; +} + +// Pure structural checks only (no `docker` CLI invocation): the self-hosted runner container this actually +// runs on does not have Docker-in-Docker access, so a test that shells out to `docker compose config` +// would be unreliable/environment-dependent here (same constraint as selfhost-compose-logging.test.ts). +// `{ merge: true }` resolves `<<: *default-logging` the same way Docker Compose's own YAML 1.1 merge-key +// support does -- verified once by hand against `docker compose config` with every profile active. +describe("docker-compose.yml — workflows + storage profiles (#1219)", () => { + const compose = readYamlWithMerge("docker-compose.yml"); + const services = record(compose.services); + const volumes = record(compose.volumes); + const n8n = record(services.n8n); + const minio = record(services.minio); + const grafana = record(services.grafana); + + it("documents the optional workflows and storage profiles in the header", () => { + const header = readFileSync("docker-compose.yml", "utf8"); + expect(header).toContain("--profile workflows"); + expect(header).toContain("--profile storage"); + expect(header).toContain("n8n workflow automation"); + expect(header).toContain("MinIO S3-compatible object storage"); + }); + + it("merges the shared logging anchor into both new services without YAML conflicts", () => { + for (const [name, service] of [ + ["n8n", n8n], + ["minio", minio], + ] as const) { + const logging = record(service).logging as { + driver?: string; + options?: Record; + }; + expect(logging.driver, name).toBe("json-file"); + expect(logging.options?.["max-size"], name).toBe("10m"); + expect(logging.options?.["max-file"], name).toBe("3"); + } + }); + + it("gates n8n behind --profile workflows with basic-auth env and a runtime password check", () => { + expect(n8n.image).toBe("n8nio/n8n:2.31.0"); + expect(n8n.profiles).toEqual(["workflows"]); + expect(n8n.ports).toEqual(["5678:5678"]); + expect(n8n.volumes).toEqual(["n8n-data:/home/node/.n8n"]); + + const env = record(n8n.environment); + expect(env.N8N_BASIC_AUTH_ACTIVE).toBe("true"); + expect(env.N8N_BASIC_AUTH_USER).toBe("${N8N_USER:-admin}"); + // Soft default (:-), not hard :? -- compose interpolates the whole file even when the profile is inactive. + expect(env.N8N_BASIC_AUTH_PASSWORD).toBe("${N8N_PASSWORD:-}"); + expect(env.WEBHOOK_URL).toBe("${N8N_WEBHOOK_URL:-http://localhost:5678}"); + expect(env.N8N_ENCRYPTION_KEY).toBe("${N8N_ENCRYPTION_KEY:-}"); + + const entrypoint = n8n.entrypoint as string[]; + expect(entrypoint).toHaveLength(3); + expect(entrypoint[0]).toBe("/bin/sh"); + expect(entrypoint[1]).toBe("-ec"); + expect(entrypoint[2]).toContain('if [ -z "$${N8N_BASIC_AUTH_PASSWORD:-}" ]; then'); + expect(entrypoint[2]).toContain("Set N8N_PASSWORD in .env before using --profile workflows."); + expect(entrypoint[2]).toContain("exit 1"); + expect(entrypoint[2]).toContain("exec /docker-entrypoint.sh"); + + // Same multi-line shell entrypoint shape as Grafana's runtime password gate (observability profile). + expect(grafana.entrypoint).toEqual( + expect.arrayContaining([ + "/bin/sh", + "-ec", + expect.stringContaining("exec /run.sh"), + ]), + ); + }); + + it("keeps compose-safe $$ escaping in the n8n entrypoint heredoc (not a bare $ for compose interpolation)", () => { + const source = readFileSync("docker-compose.yml", "utf8"); + const n8nStart = source.indexOf(" n8n:\n"); + const n8nEnd = source.indexOf(" # ── MinIO S3-compatible object storage"); + expect(n8nStart).toBeGreaterThan(-1); + expect(n8nEnd).toBeGreaterThan(n8nStart); + const n8nBlock = source.slice(n8nStart, n8nEnd); + expect(n8nBlock).toContain('$${N8N_BASIC_AUTH_PASSWORD:-}'); + expect(n8nBlock).not.toContain('if [ -z "${N8N_BASIC_AUTH_PASSWORD:-}" ];'); + }); + + it("gates MinIO behind --profile storage on the S3 API and console ports", () => { + expect(minio.image).toBe("minio/minio:RELEASE.2025-09-07T16-13-09Z"); + expect(minio.profiles).toEqual(["storage"]); + expect(minio.ports).toEqual(["9000:9000", "9001:9001"]); + expect(minio.volumes).toEqual(["minio-data:/data"]); + expect(minio.command).toContain("server"); + expect(minio.command).toContain("--console-address"); + + const env = record(minio.environment); + // Same soft-default posture as n8n/grafana/browserless -- avoids breaking default `docker compose up`. + expect(env.MINIO_ROOT_USER).toBe("${MINIO_ROOT_USER:-}"); + expect(env.MINIO_ROOT_PASSWORD).toBe("${MINIO_ROOT_PASSWORD:-}"); + }); + + it("pins workflow/storage images to explicit versions, not floating :latest tags", () => { + expect(n8n.image).not.toMatch(/:latest$/); + expect(minio.image).not.toMatch(/:latest$/); + }); + + it("declares persistent volumes for both optional services", () => { + expect(volumes["n8n-data"]).toBeDefined(); + expect(volumes["minio-data"]).toBeDefined(); + }); +}); + +describe("n8n/workflows — bundled templates (#1219)", () => { + const templateNames = [ + "review-slack-notify.json", + "gate-daily-summary.json", + "issue-auto-triage.json", + ] as const; + + it.each(templateNames)("ships importable workflow template %s", (filename) => { + const raw = readFileSync(join("n8n/workflows", filename), "utf8"); + const workflow = JSON.parse(raw) as { + name?: string; + nodes?: unknown[]; + connections?: Record; + }; + expect(workflow.name?.length).toBeGreaterThan(0); + expect(Array.isArray(workflow.nodes)).toBe(true); + expect(workflow.nodes?.length).toBeGreaterThan(0); + expect(workflow.connections).toBeTypeOf("object"); + }); + + it("includes the Slack notify starter template required by the issue", () => { + const slack = JSON.parse(readFileSync("n8n/workflows/review-slack-notify.json", "utf8")) as { + nodes: Array<{ type: string }>; + }; + expect(slack.nodes.some((n) => n.type === "n8n-nodes-base.webhook")).toBe(true); + expect(slack.nodes.some((n) => n.type === "n8n-nodes-base.slack")).toBe(true); + }); +}); + +describe(".env.example — workflows + storage vars (#1219)", () => { + const envExample = readFileSync(".env.example", "utf8"); + + it("documents n8n and MinIO profile env vars", () => { + expect(envExample).toContain("# --- n8n workflow automation (--profile workflows) ---"); + expect(envExample).toContain("# N8N_PASSWORD=changeme"); + expect(envExample).toContain("# --- MinIO S3-compatible object storage (--profile storage) ---"); + expect(envExample).toContain("# MINIO_ROOT_USER=minio"); + expect(envExample).toContain("# MINIO_ROOT_PASSWORD=changeme"); + }); + + it("documents Litestream → MinIO wiring for local backups", () => { + expect(envExample).toContain("LITESTREAM_ENDPOINT=http://minio:9000"); + expect(envExample).toContain("LITESTREAM_ACCESS_KEY_ID=${MINIO_ROOT_USER}"); + expect(envExample).toContain("LITESTREAM_SECRET_ACCESS_KEY=${MINIO_ROOT_PASSWORD}"); + }); +});