diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx index 6b36b12475..0a966b3a86 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx @@ -154,6 +154,70 @@ sudo systemctl enable --now gittensory-docker-prune.timer`} Run it manually at any time with docker system df before and after to see what it reclaimed: sh scripts/selfhost-docker-prune.sh.

+

+ This should always prune containers, images, and build cache — never + volumes. Pruning a volume deletes real application state (the database, backups, vector + index, or a runner's registration and job data), not disposable build output, so it is + never part of routine cleanup unless you intentionally want to delete that state. +

+ +

Self-hosted runner temp storage

+

+ If you run --profile runners, keep every runner job's scratch/temp writes + on the mounted runner-work volume, never the container's plain{" "} + /tmp. A container's own /tmp lives in Docker's + overlay/containerd snapshot storage — a CI job that writes high-volume temp data there + (language toolchain caches, build artifacts, ad hoc mktemp calls) grows the + host's Docker root storage directly, not the volume, so it is invisible to + volume-scoped cleanup and can fill the disk out from under the whole stack. The shipped{" "} + runner service points TMPDIR, TMP, and{" "} + TEMP at /tmp/runner/tmp (a subdirectory of the mounted{" "} + runner-work volume) and keeps RUNNER_WORKDIR at{" "} + /tmp/runner on the same volume. A one-shot runner-tmp-init service + creates that directory on the volume (and makes it world-writable, matching real{" "} + /tmp permissions) before the runner container starts, so this works out of the + box on a fresh volume with no manual steps. +

+

+ Adding a second or third runner service in docker-compose.override.yml for + higher CI throughput? Each one needs its own runner-work-style volume, its own + init step, and the same temp env — YAML anchors don't cross separate compose files, so + repeat the extension block in your override file: +

+

Sentry tracing

diff --git a/docker-compose.override.yml.example b/docker-compose.override.yml.example index 83f7070091..6976d6b23e 100644 --- a/docker-compose.override.yml.example +++ b/docker-compose.override.yml.example @@ -34,6 +34,16 @@ # The RECOMMENDED alternative to all of this: don't co-locate runners with the review stack at all -- use # GitHub-hosted CI, or a separate dedicated runner host (see the runner service's comment in # docker-compose.yml). +# +# SCALING TO MULTIPLE RUNNERS: adding a second/third runner service here (e.g. to raise CI throughput)? +# Each one MUST set the same TMPDIR/TMP/TEMP env (pointed at its own mounted runner-work-style volume, +# never left at the container's plain /tmp) and mount its own named volume at /tmp/runner -- otherwise its +# CI job temp writes fall back to the container's overlay storage (the exact growth/disk-pressure problem +# this repo's runner service is set up to avoid). YAML anchors don't cross separate compose files, so you +# cannot reference docker-compose.yml's `x-runner-tmp-env` anchor from this file directly -- either repeat +# the three env vars literally, or redeclare the SAME `x-runner-tmp-env: &runner-tmp-env` extension block +# at the top of your own override file and merge it with `<<: *runner-tmp-env`. See the self-hosting +# operations docs for a complete, copy-pasteable multi-runner service snippet. services: gittensory: diff --git a/docker-compose.yml b/docker-compose.yml index a93e6d4dd6..19c4a768a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,23 @@ x-logging: &default-logging max-size: "10m" max-file: "3" +# Self-hosted GitHub Actions runner temp env (#selfhost-runner-tmp): a container's plain /tmp lives in +# Docker's overlay/containerd snapshot storage, NOT any mounted volume -- high-volume CI scratch writes +# there (language toolchains, build caches, ad hoc `mktemp` calls) grow the HOST's Docker root storage +# directly and are invisible to volume-scoped cleanup (confirmed in production: unbounded overlay growth, +# disk pressure, and `docker system df` races against disappearing temp files). Every self-hosted-runner +# service -- the `runner` service below, merged in via `<<: *runner-tmp-env`, AND any additional runner +# service you add for a multi-runner deployment -- must point TMPDIR/TMP/TEMP at its own mounted +# runner-work-style volume the same way, so job temp writes always land on mounted storage instead of the +# container's writable layer. A second compose file (e.g. docker-compose.override.yml) CANNOT reference +# this anchor directly -- YAML anchors don't cross files -- so an override adding another runner service +# must redeclare this same x-runner-tmp-env block in its own file before merging it. See +# docker-compose.override.yml.example and the self-hosting operations docs for the full pattern. +x-runner-tmp-env: &runner-tmp-env + TMPDIR: /tmp/runner/tmp + TMP: /tmp/runner/tmp + TEMP: /tmp/runner/tmp + services: # ── Core app (always runs) ───────────────────────────────────────────────── gittensory: @@ -620,6 +637,20 @@ services: # containers on an 8-vCPU box left the app starved under load. See docker-compose.override.yml.example # for a proven CPU-priority pattern (relative cpu_shares + a per-container cpus ceiling) before scaling # this up, and ./scripts/docker-prune.sh for reclaiming the disk space CI builds accumulate. + # One-shot bootstrap for the runner's TMPDIR (below): guarantees /tmp/runner/tmp exists (and is + # world-writable, matching real /tmp's mode -- the runner image's runtime user isn't guaranteed to be + # root) on the runner-work volume BEFORE the runner container starts. A fresh named volume has nothing + # at that path otherwise, and pointing TMPDIR at a directory that doesn't exist yet would break every + # job step that shells out expecting $TMPDIR to be usable. `mkdir -p` is idempotent, so this safely + # no-ops on every subsequent `docker compose up`; it never touches the runner image's own entrypoint. + runner-tmp-init: + image: alpine:3.20 + <<: *default-logging + profiles: ["runners"] + volumes: + - runner-work:/tmp/runner + command: ["sh", "-c", "mkdir -p /tmp/runner/tmp && chmod 1777 /tmp/runner/tmp"] + runner: # Pin to a specific version tag — `latest` is mutable. Find a digest via: # docker pull myoung34/github-runner:ubuntu-jammy && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-jammy @@ -627,7 +658,11 @@ services: restart: unless-stopped <<: *default-logging profiles: ["runners"] + depends_on: + runner-tmp-init: + condition: service_completed_successfully environment: + <<: *runner-tmp-env RUNNER_SCOPE: ${RUNNER_SCOPE:-repo} REPO_URL: ${RUNNER_REPO_URL:-} RUNNER_TOKEN: ${RUNNER_TOKEN:-} diff --git a/test/unit/docs-selfhost-operations-runner-tmpdir.test.ts b/test/unit/docs-selfhost-operations-runner-tmpdir.test.ts new file mode 100644 index 0000000000..e17f5270f6 --- /dev/null +++ b/test/unit/docs-selfhost-operations-runner-tmpdir.test.ts @@ -0,0 +1,74 @@ +import { readFileSync } from "node:fs"; +import { parseDocument } from "yaml"; +import { describe, expect, it } from "vitest"; + +// Drift guard (#selfhost-runner-tmp): the self-hosting operations doc's multi-runner override snippet +// must stay in sync with docker-compose.yml's real x-runner-tmp-env anchor + runner-tmp-init pattern -- +// mirrors the same source-of-truth-diff approach as docs-selfhost-troubleshooting-metric-names.test.ts. +// If the real compose pattern ever changes without the docs snippet changing too, this fails instead of +// operators copy-pasting a stale/incorrect multi-runner example. + +const DOC_PATH = "apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx"; + +function readYamlWithMerge(path: string): Record { + const doc = parseDocument(readFileSync(path, "utf8"), { merge: true }); + const value = doc.toJS(); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${path} must be a YAML object`); + } + return value as Record; +} + +describe("self-hosting-operations doc: runner temp storage guidance matches docker-compose.yml (#selfhost-runner-tmp)", () => { + it("explains why runner temp must stay off overlay storage", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + expect(doc).toMatch(/runner-work/); + expect(doc).toMatch(/overlay/); + expect(doc).toContain("TMPDIR"); + }); + + it("mentions the runner-tmp-init bootstrap mechanism by concept, not just by name", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + expect(doc).toContain("runner-tmp-init"); + expect(doc).toMatch(/before the runner container\s+starts/); + }); + + it("says Docker cleanup should prune containers/images/build cache, never volumes", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + expect(doc).toMatch(/containers, images, and build cache/i); + expect(doc).toMatch(/never\s+volumes/i); + }); + + it("includes a multi-runner override snippet whose x-runner-tmp-env values match the real anchor in docker-compose.yml", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + const codeBlockMatch = /x-runner-tmp-env: &runner-tmp-env[\s\S]*?volumes:\n {2}runner-work-2:/.exec(doc); + expect(codeBlockMatch, "expected a multi-runner CodeBlock snippet in the doc").not.toBeNull(); + const snippet = codeBlockMatch![0]; + + const compose = readYamlWithMerge("docker-compose.yml"); + const realAnchor = compose["x-runner-tmp-env"] as Record; + expect(realAnchor.TMPDIR).toBeTruthy(); + + for (const key of ["TMPDIR", "TMP", "TEMP"] as const) { + expect(snippet, `docs snippet missing ${key}: ${realAnchor[key]}`).toContain(`${key}: ${realAnchor[key]}`); + } + }); + + it("the doc's multi-runner snippet uses the same depends_on condition as the real runner service", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + const compose = readYamlWithMerge("docker-compose.yml"); + const services = compose.services as Record>; + const runner = services.runner!; + const dependsOn = runner.depends_on as Record; + const condition = Object.values(dependsOn)[0]?.condition; + expect(condition).toBeTruthy(); + expect(doc).toContain(`condition: ${condition}`); + }); + + it("the doc's multi-runner snippet never mounts the Docker socket", () => { + const doc = readFileSync(DOC_PATH, "utf8"); + const codeBlockMatch = /x-runner-tmp-env: &runner-tmp-env[\s\S]*?volumes:\n {2}runner-work-2:/.exec(doc); + expect(codeBlockMatch).not.toBeNull(); + expect(codeBlockMatch![0]).not.toMatch(/docker\.sock/); + }); +}); diff --git a/test/unit/selfhost-compose-runner-tmpdir.test.ts b/test/unit/selfhost-compose-runner-tmpdir.test.ts new file mode 100644 index 0000000000..b885b98d22 --- /dev/null +++ b/test/unit/selfhost-compose-runner-tmpdir.test.ts @@ -0,0 +1,129 @@ +import { readFileSync } from "node:fs"; +import { parseDocument } from "yaml"; +import { describe, expect, it } from "vitest"; + +function readYamlWithMerge(path: string): Record { + const doc = parseDocument(readFileSync(path, "utf8"), { merge: true }); + const value = doc.toJS(); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${path} must be a YAML object`); + } + return value as Record; +} + +type ComposeService = Record; + +function composeServices(): Record { + const compose = readYamlWithMerge("docker-compose.yml"); + return (compose.services as Record) ?? {}; +} + +function runnerService(): ComposeService { + const services = composeServices(); + expect(services.runner, "docker-compose.yml must define a runner service").toBeTruthy(); + return services.runner as ComposeService; +} + +// Runner CI job temp writes must stay on the mounted runner-work volume, never the container's plain +// /tmp (which lives in Docker's overlay/containerd snapshot storage and grows the HOST's Docker root +// storage directly -- invisible to volume-scoped cleanup, confirmed in production as unbounded overlay +// growth, disk pressure, and `docker system df` races against disappearing temp files). Pure structural +// checks only (no `docker` CLI invocation): the self-hosted runner container this actually runs on does +// not have Docker-in-Docker access, so a test that shells out to `docker compose config` would be +// unreliable/environment-dependent here (same constraint as the other selfhost-compose-*.test.ts files). +// `{ merge: true }` resolves `<<: *runner-tmp-env` the same way Docker Compose's own YAML 1.1 merge-key +// support does -- verified once by hand against `docker compose config` with every profile active. +describe("docker-compose.yml — runner temp storage stays off overlay (#selfhost-runner-tmp)", () => { + it("mounts the runner-work volume at /tmp/runner", () => { + const runner = runnerService(); + const volumes = (runner.volumes as string[]) ?? []; + expect(volumes).toContain("runner-work:/tmp/runner"); + }); + + it("keeps RUNNER_WORKDIR on the mounted volume", () => { + const runner = runnerService(); + const env = runner.environment as Record; + expect(env.RUNNER_WORKDIR).toBe("/tmp/runner"); + }); + + it.each(["TMPDIR", "TMP", "TEMP"])("sets %s for the runner", (key) => { + const runner = runnerService(); + const env = runner.environment as Record; + expect(env[key], key).toBeDefined(); + }); + + it.each(["TMPDIR", "TMP", "TEMP"])( + "points %s at mounted runner storage, not the container's plain /tmp", + (key) => { + const runner = runnerService(); + const env = runner.environment as Record; + const value = env[key]; + expect(value, key).not.toBe("/tmp"); + expect(typeof value, key).toBe("string"); + expect(value as string, `${key}=${String(value)} must live under the mounted /tmp/runner volume`).toMatch( + /^\/tmp\/runner(\/|$)/, + ); + }, + ); + + it("never mounts the Docker socket into the runner service", () => { + const runner = runnerService(); + const volumes = (runner.volumes as string[]) ?? []; + expect(volumes.some((v) => v.includes("docker.sock"))).toBe(false); + }); + + it("guarantees the configured TMPDIR exists before the runner starts, via a depends_on init service", () => { + const services = composeServices(); + const runner = runnerService(); + const dependsOn = runner.depends_on as Record | undefined; + expect(dependsOn, "runner must declare depends_on for a bootstrap/init service").toBeTruthy(); + + const initServiceNames = Object.keys(dependsOn ?? {}); + expect(initServiceNames.length, "runner must depend on exactly the init service").toBeGreaterThan(0); + const initServiceName = initServiceNames[0]!; + expect(dependsOn?.[initServiceName]?.condition, "the dependency must wait for successful completion").toBe( + "service_completed_successfully", + ); + + const initService = services[initServiceName]; + expect(initService, `the depends_on target "${initServiceName}" must exist as a service`).toBeTruthy(); + const initVolumes = (initService?.volumes as string[]) ?? []; + expect( + initVolumes.some((v) => v.startsWith("runner-work:")), + "the init service must mount the same runner-work volume it bootstraps", + ).toBe(true); + expect(JSON.stringify(initService?.command)).toMatch(/mkdir -p/); + }); + + it("does not introduce volume-deleting behavior in the runner temp bootstrap step", () => { + const services = composeServices(); + const runner = runnerService(); + const dependsOn = runner.depends_on as Record; + const initServiceName = Object.keys(dependsOn)[0]!; + const initService = services[initServiceName]; + const command = JSON.stringify(initService?.command ?? ""); + expect(command).not.toMatch(/\brm\b|\bprune\b|\bdown\b\s+-v/); + }); + + it("keeps the runner service gated behind the runners profile (opt-in, not part of the default stack)", () => { + const runner = runnerService(); + expect(runner.profiles).toEqual(["runners"]); + }); + + it("exposes a reusable x-runner-tmp-env anchor that the runner service actually merges in", () => { + const compose = readYamlWithMerge("docker-compose.yml"); + const anchorBlock = compose["x-runner-tmp-env"] as Record; + expect(anchorBlock, "x-runner-tmp-env extension field must exist for multi-runner reuse").toBeTruthy(); + for (const key of ["TMPDIR", "TMP", "TEMP"]) { + expect(anchorBlock[key], key).toBe("/tmp/runner/tmp"); + } + + // The runner service's resolved env must equal the anchor's values (not a separately hand-written + // duplicate that could drift) -- `{ merge: true }` above already resolved `<<: *runner-tmp-env`. + const runner = runnerService(); + const env = runner.environment as Record; + for (const key of ["TMPDIR", "TMP", "TEMP"]) { + expect(env[key], key).toBe(anchorBlock[key]); + } + }); +});