feat(engine): add a load-testing harness for iterate-loop under concurrent load

package.json

@@ -20,6 +20,7 @@
     "miner:env-reference": "node packages/loopover-miner/scripts/generate-env-reference.mjs",
     "miner:env-reference:check": "node packages/loopover-miner/scripts/generate-env-reference.mjs --check",
     "benchmark:miner": "node packages/loopover-miner/scripts/benchmark.mjs",
+    "loadtest:iterate-loop": "npm run build --workspace @loopover/engine && node packages/loopover-engine/scripts/load-test-iterate-loop.mjs",
     "command-reference": "node scripts/gen-command-reference.mjs",
     "command-reference:check": "node scripts/gen-command-reference.mjs --check",
     "selfhost:validate-observability": "node scripts/validate-observability-configs.mjs",

packages/loopover-engine/docs/iterate-loop-load-test.md

@@ -0,0 +1,62 @@
+# iterate-loop load test
+
+A committed load-testing harness for `runIterateLoop` (`src/miner/iterate-loop.ts`), the create->score->
+self-review->decide orchestrator AMS runs once per attempt. It reuses the same `CodingAgentDriver`
+injection seam `iterate-loop.test.ts` already exercises — the driver never spawns a real subprocess or
+spends API budget — but adds a configurable artificial per-iteration delay, so the numbers below measure
+iterate-loop's own orchestration/scheduling overhead under concurrent, multi-tenant-like load rather than a
+network call's latency. See issue #4913 for the parallel Worker-endpoint load-testing precedent this
+mirrors; #5224 is the AMS-side counterpart this harness was built for.
+
+## Running it
+
+```sh
+npm run loadtest:iterate-loop
+# or, from a workspace checkout, after building the engine:
+npm --workspace @loopover/engine run build
+node packages/loopover-engine/scripts/load-test-iterate-loop.mjs
+```
+
+This prints a short text report to stdout and exits `0`. It does not fail the build or a CI job on its own
+— it is a signal to read, not a hard gate (there is no fixed pass/fail threshold, since wall-clock timing
+on shared CI runners is too noisy to gate on reliably). Run it locally before/after a change to
+`iterate-loop.ts`, `iterate-policy.ts`, `attempt-metering.ts`, or `self-review-adapter.ts` to see whether
+the change moved the needle under concurrency.
+
+## What it measures
+
+Each concurrency level runs a batch of simulated tenant attempts (a distinct `repoFullName`/
+`contributorLogin` per attempt, mirroring how independent tenants share the same AMS infra) through
+`runIterateLoop`, `concurrency` attempts in flight at a time via `Promise.all`, until the configured
+attempt count for that level completes. Every attempt is scripted to hand off after exactly one iteration
+(a passing self-review verdict on the first try), so the wall-clock numbers isolate the loop's own
+per-attempt overhead — driver invocation, self-review, policy decision, attempt-log append — from any
+variation in how many iterations a real attempt would take.
+
+- **Concurrency levels:** 1, 8, 32, 128 concurrent attempts.
+- **Attempts per level:** 32 (script default) / 64 (baseline capture below).
+- **Simulated driver latency:** 15ms per iteration — a stand-in for the wall-clock a real coding-agent
+  subprocess invocation would take, without actually spending any real API budget or spawning a process.
+
+## Baseline (informational only, machine-dependent)
+
+Captured on a Linux x86_64 dev container, Node.js 22.23.1, 64 attempts per level. Absolute numbers vary by
+hardware and by real driver/self-review latency — use this as a rough sense of scale and of how throughput
+scales with concurrency, not a target:
+
+```
+iterate-loop load test
+
+concurrency=1: 1023.87ms wall for 64 attempts, 63 attempts/sec, 64/64 handed off (simulated driver latency 15ms)
+concurrency=8: 151.74ms wall for 64 attempts, 422 attempts/sec, 64/64 handed off (simulated driver latency 15ms)
+concurrency=32: 65.20ms wall for 64 attempts, 982 attempts/sec, 64/64 handed off (simulated driver latency 15ms)
+concurrency=128: 40.59ms wall for 64 attempts, 1577 attempts/sec, 64/64 handed off (simulated driver latency 15ms)
+```
+
+Throughput scales roughly linearly with concurrency up to the point where the batch size matches (or
+exceeds) the attempt count per level — at that point every attempt starts in the same tick and the
+per-attempt overhead is fully parallelized, bounded only by the simulated driver latency plus the loop's own
+synchronous work per attempt. This is execution/measurement only against the existing, already-injectable
+driver seam; it does not change `runIterateLoop`'s own concurrency model. These numbers feed the per-tenant
+scheduling and queue-fairness design work in the AMS Cloud Readiness milestone — reference them there rather
+than re-measuring.

packages/loopover-engine/scripts/load-test-iterate-loop.d.mts

@@ -0,0 +1,33 @@
+import type { CodingAgentDriver, CodingAgentDriverResult, CodingAgentDriverTask } from "../src/miner/coding-agent-driver.js";
+
+export type LoadTestOptions = {
+  levels?: number[];
+  attemptCount?: number;
+  latencyMs?: number;
+};
+
+export type LoadTestLevelResult = {
+  concurrency: number;
+  attemptCount: number;
+  latencyMs: number;
+  wallMs: number;
+  handoffCount: number;
+  attemptsPerSecond: number;
+};
+
+export declare const DEFAULT_CONCURRENCY_LEVELS: number[];
+export declare const DEFAULT_ATTEMPTS_PER_LEVEL: number;
+export declare const DEFAULT_SIMULATED_DRIVER_LATENCY_MS: number;
+
+export declare function buildFakeLoadTestDriver(latencyMs: number): CodingAgentDriver & {
+  run(task: CodingAgentDriverTask): Promise<CodingAgentDriverResult>;
+};
+
+export declare function runConcurrencyLevel(
+  concurrency: number,
+  options?: LoadTestOptions,
+): Promise<LoadTestLevelResult>;
+
+export declare function runLoadTest(options?: LoadTestOptions): Promise<LoadTestLevelResult[]>;
+
+export declare function formatLoadTestReport(results: readonly LoadTestLevelResult[]): string;

packages/loopover-engine/scripts/load-test-iterate-loop.mjs

@@ -0,0 +1,144 @@
+#!/usr/bin/env node
+import { performance } from "node:perf_hooks";
+import { parseFocusManifest, runIterateLoop } from "../dist/index.js";
+
+// Load-testing harness for the AMS iterate-loop orchestrator (#5224): every existing iterate-loop test is
+// correctness-oriented (single attempt, fake driver returns instantly), so there is no signal today for how
+// runIterateLoop behaves when many tenants' attempts run concurrently against shared infra. This harness
+// reuses the SAME fake-driver injection seam iterate-loop.test.ts already exercises (a CodingAgentDriver whose
+// `run()` never spawns a real subprocess or spends API budget) but adds a configurable artificial per-iteration
+// delay, so the measured throughput reflects iterate-loop's own orchestration/scheduling overhead under
+// concurrency rather than a network call's latency. See docs/iterate-loop-load-test.md (#5224) for how to run
+// this and read the numbers, and issue #4913 for the parallel Worker-endpoint load-testing precedent.
+
+export const DEFAULT_CONCURRENCY_LEVELS = [1, 8, 32, 128];
+export const DEFAULT_ATTEMPTS_PER_LEVEL = 32;
+export const DEFAULT_SIMULATED_DRIVER_LATENCY_MS = 15;
+
+const SYNTHETIC_ISSUE_NUMBER = 7;
+
+/** One open issue per synthetic tenant repo, matching that tenant's own `passesPredictedGate` linkage below --
+ *  each tenant is a fully independent repo/contributor pair (`buildSelfReviewPredictedGateInput`'s own identity
+ *  fields), the same "multi-tenant-like" shape the issue's Problem section asks this harness to load-test. */
+function buildReviewContext(tenantRepoFullName) {
+  return {
+    manifest: parseFocusManifest({ gate: { duplicates: "block", linkedIssue: "advisory" } }),
+    repo: { fullName: tenantRepoFullName, owner: tenantRepoFullName.split("/")[0], name: tenantRepoFullName.split("/")[1], isInstalled: true, isRegistered: true, isPrivate: false },
+    issues: [{ repoFullName: tenantRepoFullName, number: SYNTHETIC_ISSUE_NUMBER, title: "Synthetic load-test issue", state: "open", labels: [], linkedPrs: [] }],
+    pullRequests: [],
+  };
+}
+
+/** A `CodingAgentDriver` (coding-agent-driver.ts) that never spawns a real subprocess or spends API budget --
+ *  it resolves after `latencyMs` (simulating the wall-clock an iteration of a real coding-agent invocation
+ *  would take) with a scripted, always-passing result. `latencyMs` uses a real `setTimeout`, not a busy-loop, so
+ *  concurrent attempts genuinely interleave on the event loop the way concurrent live attempts would. */
+export function buildFakeLoadTestDriver(latencyMs) {
+  return {
+    async run(task) {
+      await new Promise((resolve) => setTimeout(resolve, latencyMs));
+      return { ok: true, changedFiles: [`src/${task.attemptId}.ts`], summary: `synthetic load-test change for ${task.attemptId}`, turnsUsed: 1 };
+    },
+  };
+}
+
+const NOOP_SLOP_ASSESSMENT = { slopRisk: 0, band: "clean", findings: [] };
+
+/** One simulated tenant attempt: a distinct `repoFullName`/`contributorLogin` per `tenantIndex`, a linked open
+ *  issue that matches on the first iteration, so every attempt hands off in exactly one iteration -- isolating
+ *  the measurement to iterate-loop's own per-attempt orchestration overhead rather than varying iteration counts
+ *  across runs. */
+async function runOneAttempt(tenantIndex, driver) {
+  const repoFullName = `load-test-tenant-${tenantIndex}/repo`;
+  const attemptId = `attempt-${tenantIndex}`;
+  const input = {
+    attemptId,
+    workingDirectory: `/tmp/${attemptId}`,
+    acceptanceCriteriaPath: `/tmp/${attemptId}/acceptance-criteria.json`,
+    instructions: "Synthetic load-test instructions",
+    mode: "live",
+    maxIterations: 3,
+    maxTurnsPerIteration: 20,
+    repoFullName,
+    contributorLogin: `miner-${tenantIndex}`,
+    title: "Synthetic load-test attempt",
+    body: `Closes #${SYNTHETIC_ISSUE_NUMBER}`,
+    linkedIssues: [SYNTHETIC_ISSUE_NUMBER],
+    reviewContext: buildReviewContext(repoFullName),
+    rejectionSignaled: false,
+  };
+  const deps = {
+    driver,
+    runSlopAssessment: () => NOOP_SLOP_ASSESSMENT,
+    appendAttemptLogEvent: () => {},
+  };
+  const start = performance.now();
+  const result = await runIterateLoop(input, deps);
+  return { elapsedMs: performance.now() - start, result };
+}
+
+/**
+ * Run `attemptCount` simulated tenant attempts concurrently (`Promise.all`, all started in the same tick) against
+ * one shared fake driver, and report the aggregate wall time plus derived throughput. Every attempt is expected
+ * to hand off after its first iteration (see {@link runOneAttempt}) -- a non-`"handoff"` outcome or a driver
+ * error would silently understate real concurrent load, so both are counted and surfaced rather than ignored.
+ */
+export async function runConcurrencyLevel(concurrency, options = {}) {
+  const attemptCount = options.attemptCount ?? DEFAULT_ATTEMPTS_PER_LEVEL;
+  const latencyMs = options.latencyMs ?? DEFAULT_SIMULATED_DRIVER_LATENCY_MS;
+  const driver = buildFakeLoadTestDriver(latencyMs);
+
+  const start = performance.now();
+  const outcomes = [];
+  for (let batchStart = 0; batchStart < attemptCount; batchStart += concurrency) {
+    const batchSize = Math.min(concurrency, attemptCount - batchStart);
+    const batch = await Promise.all(
+      Array.from({ length: batchSize }, (_unused, offset) => runOneAttempt(batchStart + offset, driver)),
+    );
+    outcomes.push(...batch);
+  }
+  const wallMs = performance.now() - start;
+  const handoffCount = outcomes.filter((o) => o.result.outcome === "handoff").length;
+
+  return {
+    concurrency,
+    attemptCount,
+    latencyMs,
+    wallMs,
+    handoffCount,
+    attemptsPerSecond: attemptCount / (wallMs / 1000),
+  };
+}
+
+/** Run every concurrency level in `levels` in sequence (never overlapping each other), so one level's
+ *  scheduling contention never bleeds into the next level's measurement. */
+export async function runLoadTest(options = {}) {
+  const levels = options.levels ?? DEFAULT_CONCURRENCY_LEVELS;
+  const results = [];
+  for (const concurrency of levels) {
+    results.push(await runConcurrencyLevel(concurrency, options));
+  }
+  return results;
+}
+
+/** Render load-test results as a stable, greppable text report (no locale-dependent number formatting). */
+export function formatLoadTestReport(results) {
+  const lines = ["iterate-loop load test", ""];
+  for (const r of results) {
+    lines.push(
+      `concurrency=${r.concurrency}: ${r.wallMs.toFixed(2)}ms wall for ${r.attemptCount} attempts, ` +
+        `${Math.round(r.attemptsPerSecond)} attempts/sec, ${r.handoffCount}/${r.attemptCount} handed off ` +
+        `(simulated driver latency ${r.latencyMs}ms)`,
+    );
+  }
+  return lines.join("\n");
+}
+
+async function main() {
+  const results = await runLoadTest();
+  console.log(formatLoadTestReport(results));
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}

test/unit/iterate-loop-load-test-script.test.ts

@@ -0,0 +1,89 @@
+import { performance } from "node:perf_hooks";
+import { spawnSync } from "node:child_process";
+import { describe, expect, it } from "vitest";
+import {
+  DEFAULT_ATTEMPTS_PER_LEVEL,
+  DEFAULT_CONCURRENCY_LEVELS,
+  DEFAULT_SIMULATED_DRIVER_LATENCY_MS,
+  buildFakeLoadTestDriver,
+  formatLoadTestReport,
+  runConcurrencyLevel,
+  runLoadTest,
+} from "../../packages/loopover-engine/scripts/load-test-iterate-loop.mjs";
+
+describe("iterate-loop load-test script (#5224)", () => {
+  it("the fake driver never spawns a real subprocess and resolves a scripted ok result after the configured latency", async () => {
+    const driver = buildFakeLoadTestDriver(5);
+    const start = performance.now();
+    const result = await driver.run({
+      attemptId: "attempt-0",
+      workingDirectory: "/tmp/attempt-0",
+      acceptanceCriteriaPath: "/tmp/attempt-0/acceptance-criteria.json",
+      instructions: "synthetic",
+      maxTurns: 1,
+    });
+    expect(performance.now() - start).toBeGreaterThanOrEqual(4);
+    expect(result.ok).toBe(true);
+    expect(result.changedFiles).toEqual(["src/attempt-0.ts"]);
+    expect(result.turnsUsed).toBe(1);
+  });
+
+  it("runs a small concurrency level end-to-end and every attempt hands off on its first iteration", async () => {
+    const level = await runConcurrencyLevel(4, { attemptCount: 8, latencyMs: 1 });
+    expect(level.concurrency).toBe(4);
+    expect(level.attemptCount).toBe(8);
+    expect(level.latencyMs).toBe(1);
+    expect(level.handoffCount).toBe(8);
+    expect(Number.isFinite(level.wallMs)).toBe(true);
+    expect(level.wallMs).toBeGreaterThan(0);
+    expect(Number.isFinite(level.attemptsPerSecond)).toBe(true);
+    expect(level.attemptsPerSecond).toBeGreaterThan(0);
+  });
+
+  it("runs a concurrency level where the batch size exceeds the attempt count in a single batch", async () => {
+    const level = await runConcurrencyLevel(128, { attemptCount: 3, latencyMs: 1 });
+    expect(level.attemptCount).toBe(3);
+    expect(level.handoffCount).toBe(3);
+  });
+
+  it("runs every concurrency level supplied via options.levels, in order", async () => {
+    const results = await runLoadTest({ levels: [1, 2], attemptCount: 2, latencyMs: 1 });
+    expect(results).toHaveLength(2);
+    expect(results.map((r) => r.concurrency)).toEqual([1, 2]);
+    for (const r of results) expect(r.handoffCount).toBe(2);
+  });
+
+  it("exposes the documented default concurrency levels, attempt count, and simulated latency", () => {
+    expect(DEFAULT_CONCURRENCY_LEVELS).toEqual([1, 8, 32, 128]);
+    expect(DEFAULT_ATTEMPTS_PER_LEVEL).toBe(32);
+    expect(DEFAULT_SIMULATED_DRIVER_LATENCY_MS).toBe(15);
+  });
+
+  it("renders a deterministic report with no locale-dependent number formatting", () => {
+    expect(
+      formatLoadTestReport([
+        { concurrency: 1, attemptCount: 10, latencyMs: 15, wallMs: 160.4, handoffCount: 10, attemptsPerSecond: 62.34 },
+        { concurrency: 8, attemptCount: 10, latencyMs: 15, wallMs: 20.1, handoffCount: 9, attemptsPerSecond: 497.5 },
+      ]),
+    ).toBe(
+      [
+        "iterate-loop load test",
+        "",
+        "concurrency=1: 160.40ms wall for 10 attempts, 62 attempts/sec, 10/10 handed off (simulated driver latency 15ms)",
+        "concurrency=8: 20.10ms wall for 10 attempts, 498 attempts/sec, 9/10 handed off (simulated driver latency 15ms)",
+      ].join("\n"),
+    );
+  });
+
+  it("runs end-to-end as a CLI script and prints the report header plus every default concurrency level", () => {
+    const result = spawnSync(process.execPath, ["packages/loopover-engine/scripts/load-test-iterate-loop.mjs"], {
+      cwd: process.cwd(),
+      encoding: "utf8",
+    });
+    expect(result.status).toBe(0);
+    expect(result.stdout).toContain("iterate-loop load test");
+    for (const level of DEFAULT_CONCURRENCY_LEVELS) {
+      expect(result.stdout).toContain(`concurrency=${level}:`);
+    }
+  });
+});