From 500dbcc174c7c8cfdfd2e01018a82529a511ffd3 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 2 Jul 2026 02:33:01 -0700 Subject: [PATCH] chore(selfhost): automate Docker disk hygiene for the self-host stack Live production observation showed disk usage climbing to a majority-full state, with unused Docker images and build cache as the dominant reclaimable consumers, and no automated mitigation anywhere in the repo -- no log rotation config on any of the 22 compose services, and no scheduled prune of Docker resources. An unaddressed disk-exhaustion path is a hard outage for a 24/7 stack: Postgres writes fail, log writes fail, and deploys fail once the volume fills. - Add a shared `x-logging` anchor (json-file, 10MB x 3 files) merged into every service in docker-compose.yml via `<<: *default-logging`, so none of them default to Docker's unbounded log driver. - Add scripts/selfhost-docker-prune.sh: age-filtered (7-day default) `docker image prune` + `docker builder prune`, paired with host-level systemd timer/service .example units (following the repo's existing .env.example templating convention) -- this runs on the HOST, not as a compose service, because reclaiming images/build-cache needs real Docker daemon access, which this repo deliberately does not grant to any container (matching the existing docker-proxy/runner services' documented security posture). - Document both as a required self-hosting-operations step. Validation: docker compose config with every profile active resolves a bounded logging config on all 22 services (verified programmatically); the prune script is tested by execution with a stubbed `docker` binary on PATH, confirming both prune calls always carry an age filter and the retention window is operator-tunable via env var. Full local gate green; no src/** files touched, so no Codecov patch-coverage obligation. --- .../routes/docs.self-hosting-operations.tsx | 24 ++++++ docker-compose.yml | 35 +++++++++ scripts/selfhost-docker-prune.sh | 30 ++++++++ .../gittensory-docker-prune.service.example | 27 +++++++ systemd/gittensory-docker-prune.timer.example | 13 ++++ test/unit/selfhost-compose-logging.test.ts | 46 ++++++++++++ .../unit/selfhost-docker-prune-script.test.ts | 75 +++++++++++++++++++ 7 files changed, 250 insertions(+) create mode 100644 scripts/selfhost-docker-prune.sh create mode 100644 systemd/gittensory-docker-prune.service.example create mode 100644 systemd/gittensory-docker-prune.timer.example create mode 100644 test/unit/selfhost-compose-logging.test.ts create mode 100644 test/unit/selfhost-docker-prune-script.test.ts diff --git a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx index 8f372ebbe6..e62f18ed74 100644 --- a/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx +++ b/apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx @@ -121,6 +121,30 @@ docker compose --profile postgres --profile observability --profile backup up -d Dead jobs stay at zero routine check below is watching for.

+

Docker resource hygiene

+

+ Every service in docker-compose.yml caps its own container logs (10MB × 3 + rotated files) out of the box, so log growth alone won't fill your disk. Unused Docker + images and build cache are a separate, larger disk-growth vector on a host that rebuilds or + pulls images repeatedly over months — Docker does not reclaim either automatically. +

+

+ Install the provided host-level timer to reclaim both on a schedule (anything unused for + less than 7 days is left alone, so a recent deploy is never at risk): +

+ +

+ Run it manually at any time with docker system df before and after to see what + it reclaimed: sh scripts/selfhost-docker-prune.sh. +

+

Sentry tracing

Leave SENTRY_TRACES_SAMPLE_RATE unset or blank to disable trace export, or set diff --git a/docker-compose.yml b/docker-compose.yml index 60697dd0e9..148f2a563e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,6 +22,19 @@ # docker compose --profile observability up -d # metrics + logs + dashboards # docker compose --profile tailscale --profile runners up -d # tailnet + CI runners +# Bounded container logging (#audit-rate-headroom): every service below defaults to Docker's +# json-file driver, which has NO size cap on its own -- a long-running 24/7 stack can fill the +# host disk purely from log growth (the app, Postgres, Redis, and the CI runners are the biggest +# producers). This anchor caps each service to 3 rotated files of 10MB (30MB/service ceiling, +# ~600MB worst case across every service in this file) and is merged into every service via +# `<<: *default-logging`. Override per-service if you need more retained log history. +x-logging: &default-logging + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + services: # ── Core app (always runs) ───────────────────────────────────────────────── gittensory: @@ -31,6 +44,7 @@ services: INSTALL_AI_CLIS: "${INSTALL_AI_CLIS:-true}" INSTALL_VISUAL_REVIEW: "${INSTALL_VISUAL_REVIEW:-false}" restart: unless-stopped + <<: *default-logging ports: # Remove this when using the caddy profile — Caddy becomes the public listener. - "${PORT:-8787}:8787" @@ -121,6 +135,7 @@ services: redis: image: redis:7-alpine restart: unless-stopped + <<: *default-logging command: - redis-server - --maxmemory @@ -140,6 +155,7 @@ services: postgres: image: pgvector/pgvector:pg16 restart: unless-stopped + <<: *default-logging profiles: ["postgres", "pgbouncer"] environment: POSTGRES_USER: gittensory @@ -158,6 +174,7 @@ services: pgbouncer: image: edoburu/pgbouncer:v1.25.2-p0 restart: unless-stopped + <<: *default-logging profiles: ["pgbouncer"] depends_on: postgres: @@ -188,6 +205,7 @@ services: postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter:v0.20.0 restart: unless-stopped + <<: *default-logging profiles: ["postgres", "pgbouncer"] depends_on: postgres: @@ -206,6 +224,7 @@ services: qdrant: image: qdrant/qdrant:v1.18.2 restart: unless-stopped + <<: *default-logging profiles: ["qdrant"] # Ports are bound to LOOPBACK (127.0.0.1), not 0.0.0.0: the app reaches Qdrant over the internal # docker network (QDRANT_URL=http://qdrant:6333), so the host mapping exists only for the local @@ -240,6 +259,7 @@ services: ollama: image: ollama/ollama:0.30.10 restart: unless-stopped + <<: *default-logging profiles: ["ollama"] volumes: - ollama-models:/root/.ollama @@ -258,6 +278,7 @@ services: litestream: image: litestream/litestream:0.5.12 restart: unless-stopped + <<: *default-logging profiles: ["litestream"] command: replicate depends_on: @@ -278,6 +299,7 @@ services: caddy: image: caddy:2-alpine restart: unless-stopped + <<: *default-logging profiles: ["caddy"] ports: - "80:80" @@ -299,6 +321,7 @@ services: prometheus: image: prom/prometheus:v3.12.0 restart: unless-stopped + <<: *default-logging profiles: ["observability"] volumes: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro @@ -314,6 +337,7 @@ services: alertmanager: image: prom/alertmanager:v0.33.0 restart: unless-stopped + <<: *default-logging profiles: ["observability"] depends_on: [prometheus] expose: @@ -328,6 +352,7 @@ services: grafana: image: grafana/grafana:13.1.0 restart: unless-stopped + <<: *default-logging profiles: ["observability"] depends_on: prometheus: @@ -365,6 +390,7 @@ services: reporting-exporter: image: alpine:3.20 restart: unless-stopped + <<: *default-logging profiles: ["observability"] depends_on: gittensory: @@ -414,6 +440,7 @@ services: loki: image: grafana/loki:3.7.3 restart: unless-stopped + <<: *default-logging profiles: ["observability"] command: ["-config.file=/etc/loki/loki-config.yml"] volumes: @@ -437,6 +464,7 @@ services: docker-proxy: image: tecnativa/docker-socket-proxy:0.3.0 restart: unless-stopped + <<: *default-logging profiles: ["observability"] environment: CONTAINERS: "1" # GET /containers/* (list, inspect, logs) @@ -449,6 +477,7 @@ services: promtail: image: grafana/promtail:3.6.11 restart: unless-stopped + <<: *default-logging profiles: ["observability"] command: [ @@ -475,6 +504,7 @@ services: otel-collector: image: otel/opentelemetry-collector-contrib:0.155.0 restart: unless-stopped + <<: *default-logging profiles: ["observability"] depends_on: tempo: @@ -489,6 +519,7 @@ services: tempo: image: grafana/tempo:2.6.1 restart: unless-stopped + <<: *default-logging profiles: ["observability"] command: ["-config.file=/etc/tempo/tempo.yaml"] volumes: @@ -512,6 +543,7 @@ services: # --format='{{index .RepoDigests 0}}' ghcr.io/tailscale/tailscale:stable image: ghcr.io/tailscale/tailscale:stable restart: unless-stopped + <<: *default-logging profiles: ["tailscale"] hostname: gittensory cap_add: @@ -538,6 +570,7 @@ services: # docker pull myoung34/github-runner:ubuntu-jammy && docker inspect --format='{{index .RepoDigests 0}}' myoung34/github-runner:ubuntu-jammy image: myoung34/github-runner:ubuntu-jammy restart: unless-stopped + <<: *default-logging profiles: ["runners"] environment: RUNNER_SCOPE: ${RUNNER_SCOPE:-repo} @@ -562,6 +595,7 @@ services: backup: image: alpine:3.20 restart: unless-stopped + <<: *default-logging profiles: ["backup"] environment: DATABASE_PATH: /data/gittensory.sqlite @@ -601,6 +635,7 @@ services: backup-exporter: image: alpine:3.20 restart: unless-stopped + <<: *default-logging profiles: ["backup"] volumes: - gittensory-backups:/backups:ro diff --git a/scripts/selfhost-docker-prune.sh b/scripts/selfhost-docker-prune.sh new file mode 100644 index 0000000000..cb3fe083fd --- /dev/null +++ b/scripts/selfhost-docker-prune.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Automated Docker resource hygiene for a 24/7 self-hosted gittensory stack (#audit-rate-headroom). Runs on +# the HOST (via the systemd timer in systemd/gittensory-docker-prune.{service,timer}.example), not as a +# compose service: reclaiming unused images and build cache needs real Docker daemon access, which this +# repo deliberately does not grant to any container (see docker-compose.yml's docker-proxy and runner +# service comments on why raw docker.sock exposure into a container is avoided). +# +# Age-filtered so nothing built/pulled recently is touched -- a rollback within the retention window still +# has its image available. `docker image prune -a` and `docker builder prune` only ever remove resources +# Docker itself already reports as unused (a running container's own image, or an active build-cache entry +# a build is currently using, are never candidates) -- this script does not change that safety property, it +# only adds the age floor on top of it. +set -eu + +RETAIN_HOURS=${GITTENSORY_DOCKER_PRUNE_RETAIN_HOURS:-168} # 7 days + +echo "[docker-prune] $(date -u +%FT%TZ) starting (retain: ${RETAIN_HOURS}h)" +echo "[docker-prune] before:" +docker system df + +echo "[docker-prune] pruning unused images older than ${RETAIN_HOURS}h..." +docker image prune -af --filter "until=${RETAIN_HOURS}h" + +echo "[docker-prune] pruning build cache older than ${RETAIN_HOURS}h..." +docker builder prune -af --filter "until=${RETAIN_HOURS}h" + +echo "[docker-prune] after:" +docker system df + +echo "[docker-prune] $(date -u +%FT%TZ) done" diff --git a/systemd/gittensory-docker-prune.service.example b/systemd/gittensory-docker-prune.service.example new file mode 100644 index 0000000000..a88ee988bd --- /dev/null +++ b/systemd/gittensory-docker-prune.service.example @@ -0,0 +1,27 @@ +# Runs scripts/selfhost-docker-prune.sh on a schedule (paired with gittensory-docker-prune.timer.example) to +# reclaim unused Docker images and build cache before disk fills up on a long-running self-host deployment. +# +# Install (adjust the path below to wherever you cloned/deployed gittensory): +# sudo cp systemd/gittensory-docker-prune.service.example /etc/systemd/system/gittensory-docker-prune.service +# sudo cp systemd/gittensory-docker-prune.timer.example /etc/systemd/system/gittensory-docker-prune.timer +# sudo $EDITOR /etc/systemd/system/gittensory-docker-prune.service # fix WorkingDirectory / ExecStart path +# sudo systemctl daemon-reload +# sudo systemctl enable --now gittensory-docker-prune.timer +# +# This is a HOST-level unit, not a Docker Compose service: reclaiming images/build-cache needs real Docker +# daemon access, which no container in docker-compose.yml is granted (see that file's docker-proxy and +# runner service comments for why raw /var/run/docker.sock exposure into a container is avoided). + +[Unit] +Description=Gittensory self-host Docker resource hygiene (image + build-cache prune) +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +# REQUIRED: point this at wherever you cloned gittensory. +WorkingDirectory=/opt/gittensory +ExecStart=/bin/sh /opt/gittensory/scripts/selfhost-docker-prune.sh +# Optional: override the default 7-day (168h) safety window before an unused image/build-cache entry is +# eligible for removal. +# Environment=GITTENSORY_DOCKER_PRUNE_RETAIN_HOURS=168 diff --git a/systemd/gittensory-docker-prune.timer.example b/systemd/gittensory-docker-prune.timer.example new file mode 100644 index 0000000000..7fb52f8459 --- /dev/null +++ b/systemd/gittensory-docker-prune.timer.example @@ -0,0 +1,13 @@ +# Pairs with gittensory-docker-prune.service.example. See that file for install instructions. + +[Unit] +Description=Run gittensory-docker-prune.service daily + +[Timer] +OnCalendar=daily +Persistent=true +# Spread the run over a window instead of firing at exactly midnight on every host. +RandomizedDelaySec=30m + +[Install] +WantedBy=timers.target diff --git a/test/unit/selfhost-compose-logging.test.ts b/test/unit/selfhost-compose-logging.test.ts new file mode 100644 index 0000000000..645e8233d3 --- /dev/null +++ b/test/unit/selfhost-compose-logging.test.ts @@ -0,0 +1,46 @@ +import { readFileSync } from "node:fs"; +import { parseDocument } from "yaml"; +import { describe, expect, it } from "vitest"; + +function readYamlWithMerge(path: string): Record { + const doc = parseDocument(readFileSync(path, "utf8"), { merge: true }); + const value = doc.toJS(); + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${path} must be a YAML object`); + } + return value as Record; +} + +// Pure structural checks only (no `docker` CLI invocation): the self-hosted runner container this actually +// runs on does not have Docker-in-Docker access, so a test that shells out to `docker compose config` +// would be unreliable/environment-dependent here (same constraint as the other selfhost-compose-*.test.ts +// files). `{ merge: true }` makes the `yaml` package resolve `<<: *default-logging` the same way Docker +// Compose's own YAML 1.1 merge-key support does -- verified once by hand against `docker compose config` +// with every profile active before this test was written. +describe("docker-compose.yml — bounded container logging (#audit-rate-headroom)", () => { + it("caps every service's logs via the shared x-logging anchor, so none defaults to Docker's unbounded json-file driver", () => { + const compose = readYamlWithMerge("docker-compose.yml"); + const services = (compose.services as Record>) ?? {}; + const serviceNames = Object.keys(services); + + // Guard against a future service quietly skipping the anchor (e.g. a copy-pasted block that dropped the + // merge key) -- every single service must resolve a bounded logging config, not just a sample of them. + expect(serviceNames.length).toBeGreaterThan(15); + for (const name of serviceNames) { + const logging = services[name]?.logging as { driver?: string; options?: Record } | undefined; + expect(logging, `service "${name}" is missing a logging config`).toBeDefined(); + expect(logging?.driver).toBe("json-file"); + expect(logging?.options?.["max-size"]).toBe("10m"); + expect(logging?.options?.["max-file"]).toBe("3"); + } + }); + + it("defines the shared anchor once with a sane cap, so a future edit only needs to change one place", () => { + const compose = readYamlWithMerge("docker-compose.yml"); + const shared = compose["x-logging"] as { logging?: { driver?: string; options?: Record } }; + + expect(shared?.logging?.driver).toBe("json-file"); + expect(shared?.logging?.options?.["max-size"]).toBe("10m"); + expect(shared?.logging?.options?.["max-file"]).toBe("3"); + }); +}); diff --git a/test/unit/selfhost-docker-prune-script.test.ts b/test/unit/selfhost-docker-prune-script.test.ts new file mode 100644 index 0000000000..474b8b642a --- /dev/null +++ b/test/unit/selfhost-docker-prune-script.test.ts @@ -0,0 +1,75 @@ +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tmpRoots: string[] = []; + +function tmpRoot(): string { + const dir = mkdtempSync(join(tmpdir(), "gittensory-docker-prune-")); + tmpRoots.push(dir); + return dir; +} + +// Stubs `docker` on PATH with a fake binary that just records every invocation's arguments (one line per +// call) instead of touching a real Docker daemon -- the self-hosted runner this suite actually runs on has +// no Docker-in-Docker access, so a test that shells out to a real `docker image prune` would be +// unreliable/environment-dependent (same constraint as the compose-file structural tests). +function stubDocker(root: string): { logFile: string; binDir: string } { + const binDir = join(root, "bin"); + mkdirSync(binDir, { recursive: true }); + const logFile = join(root, "docker-calls.log"); + writeFileSync( + join(binDir, "docker"), + ["#!/bin/sh", `echo "$@" >> "${logFile}"`, "echo 'TYPE TOTAL SIZE RECLAIMABLE'", "exit 0", ""].join("\n"), + { mode: 0o755 }, + ); + chmodSync(join(binDir, "docker"), 0o755); + return { logFile, binDir }; +} + +function runPruneScript(root: string, env: Record = {}): string { + const { logFile, binDir } = stubDocker(root); + execFileSync("sh", ["scripts/selfhost-docker-prune.sh"], { + cwd: process.cwd(), + env: { ...process.env, PATH: `${binDir}:${process.env.PATH ?? ""}`, ...env }, + }); + return readFileSync(logFile, "utf8"); +} + +afterEach(() => { + for (const dir of tmpRoots.splice(0)) rmSync(dir, { force: true, recursive: true }); +}); + +describe("selfhost-docker-prune.sh", () => { + it("prunes images and build cache with the default 7-day (168h) age floor, never a blind full wipe", () => { + const calls = runPruneScript(tmpRoot()); + + expect(calls).toContain("image prune -af --filter until=168h"); + expect(calls).toContain("builder prune -af --filter until=168h"); + // Both prune calls must always carry an `until=` filter -- a bare `docker image prune -af` (no filter) + // would also remove something built moments ago, defeating the rollback-safety window. + for (const line of calls.trim().split("\n")) { + if (line.includes("prune")) expect(line).toMatch(/--filter until=\d+h/); + } + }); + + it("honors GITTENSORY_DOCKER_PRUNE_RETAIN_HOURS to widen or narrow the safety window", () => { + const calls = runPruneScript(tmpRoot(), { GITTENSORY_DOCKER_PRUNE_RETAIN_HOURS: "24" }); + + expect(calls).toContain("image prune -af --filter until=24h"); + expect(calls).toContain("builder prune -af --filter until=24h"); + expect(calls).not.toContain("168h"); + }); + + it("reports before/after docker system df around the prune calls, for the log line an operator actually reads", () => { + const calls = runPruneScript(tmpRoot()); + const invocations = calls.trim().split("\n"); + + // "system df" (no prune flags) must appear before AND after the two prune calls, so an operator watching + // logs can see what was actually reclaimed. + const dfCalls = invocations.filter((line) => line === "system df"); + expect(dfCalls).toHaveLength(2); + }); +});