From 67400772be150357b8abcd7a2ed5959fb8bb8fde Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Mon, 6 Jul 2026 23:10:02 -0700 Subject: [PATCH] fix(selfhost): add default memory limit to the runner compose service The runner service (--profile runners) was the sole service in docker-compose.yml with no deploy.resources.limits, despite the file's own comment documenting a confirmed production incident: uncapped runner containers starving the main app under load. Give it the same operator-overridable memory-limit default every other service already has, and extend the resource-limits test so this can't regress. Closes #3893 --- .env.example | 1 + docker-compose.yml | 9 +++++++++ test/unit/selfhost-compose-resource-limits.test.ts | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index b231263faf..8c35898486 100644 --- a/.env.example +++ b/.env.example @@ -394,6 +394,7 @@ REDIS_URL=redis://redis:6379 # REQUIRED for the self-host review # RUNNER_SCOPE=repo # repo | org | enterprise # RUNNER_NAME=gittensory-runner # RUNNER_LABELS=self-hosted,linux +# RUNNER_MEM_LIMIT=2g # per-runner-container memory ceiling; raise for memory-heavy CI jobs # --- Docker disk hygiene (#audit-rate-headroom / #selfhost-runtime-pressure) --- # Build cache and unused images accumulate fast on a box that builds from source or runs CI runners; a root diff --git a/docker-compose.yml b/docker-compose.yml index b7752eded6..37e4822079 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -802,6 +802,15 @@ services: # /var/run/docker.sock grants container-escape risk and is intentionally omitted. volumes: - runner-work:/tmp/runner + # A memory ceiling so a single runaway CI job can't exhaust host RAM and take down the app alongside + # it (#3893) -- same universal-default treatment every other service in this file already has. This + # is independent of the CPU-priority tuning above: that's host-specific (vCPU count, replica count) + # and stays an opt-in docker-compose.override.yml pattern, but a memory cap needs no host-specific + # sizing to be a safe default. + deploy: + resources: + limits: + memory: "${RUNNER_MEM_LIMIT:-2g}" # ── Backups (--profile backup) ──────────────────────────────────────────── # Active database backup (Postgres pg_dump or WAL-safe SQLite online backup) + a Qdrant snapshot, on a loop diff --git a/test/unit/selfhost-compose-resource-limits.test.ts b/test/unit/selfhost-compose-resource-limits.test.ts index 29ccc36500..142e6e643b 100644 --- a/test/unit/selfhost-compose-resource-limits.test.ts +++ b/test/unit/selfhost-compose-resource-limits.test.ts @@ -13,7 +13,7 @@ function readYaml(path: string): Record { // Pure structural checks only (no `docker` CLI invocation): the self-hosted runner container this actually // runs on does not have Docker-in-Docker access, so a test that shells out to `docker compose config` // would be unreliable/environment-dependent here (same constraint as docker-compose-override-example.test.ts). -describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () => { +describe("docker-compose.yml — per-service memory limits (#1828, #2495, #3893)", () => { const EXPECTED_LIMITS: Record = { gittensory: "${GITTENSORY_MEM_LIMIT:-2g}", redis: "${REDIS_MEM_LIMIT:-512m}", @@ -24,6 +24,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () = loki: "${LOKI_MEM_LIMIT:-1g}", tempo: "${TEMPO_MEM_LIMIT:-1g}", grafana: "${GRAFANA_MEM_LIMIT:-512m}", + runner: "${RUNNER_MEM_LIMIT:-2g}", }; it("caps the core app and every heavyweight optional service with an operator-overridable memory limit", () => { @@ -51,6 +52,7 @@ describe("docker-compose.yml — per-service memory limits (#1828, #2495)", () = "LOKI_MEM_LIMIT", "TEMPO_MEM_LIMIT", "GRAFANA_MEM_LIMIT", + "RUNNER_MEM_LIMIT", ]) { expect(env, key).toContain(key); }