diff --git a/package.json b/package.json index e5be51df81..0b842df771 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "miner:env-reference": "node packages/loopover-miner/scripts/generate-env-reference.mjs", "miner:env-reference:check": "node packages/loopover-miner/scripts/generate-env-reference.mjs --check", "benchmark:miner": "node packages/loopover-miner/scripts/benchmark.mjs", + "loadtest:iterate-loop": "npm run build --workspace @loopover/engine && node packages/loopover-engine/scripts/load-test-iterate-loop.mjs", "command-reference": "node scripts/gen-command-reference.mjs", "command-reference:check": "node scripts/gen-command-reference.mjs --check", "selfhost:validate-observability": "node scripts/validate-observability-configs.mjs", diff --git a/packages/loopover-engine/docs/iterate-loop-load-test.md b/packages/loopover-engine/docs/iterate-loop-load-test.md new file mode 100644 index 0000000000..9afb55f6bc --- /dev/null +++ b/packages/loopover-engine/docs/iterate-loop-load-test.md @@ -0,0 +1,62 @@ +# iterate-loop load test + +A committed load-testing harness for `runIterateLoop` (`src/miner/iterate-loop.ts`), the create->score-> +self-review->decide orchestrator AMS runs once per attempt. It reuses the same `CodingAgentDriver` +injection seam `iterate-loop.test.ts` already exercises — the driver never spawns a real subprocess or +spends API budget — but adds a configurable artificial per-iteration delay, so the numbers below measure +iterate-loop's own orchestration/scheduling overhead under concurrent, multi-tenant-like load rather than a +network call's latency. See issue #4913 for the parallel Worker-endpoint load-testing precedent this +mirrors; #5224 is the AMS-side counterpart this harness was built for. + +## Running it + +```sh +npm run loadtest:iterate-loop +# or, from a workspace checkout, after building the engine: +npm --workspace @loopover/engine run build +node packages/loopover-engine/scripts/load-test-iterate-loop.mjs +``` + +This prints a short text report to stdout and exits `0`. It does not fail the build or a CI job on its own +— it is a signal to read, not a hard gate (there is no fixed pass/fail threshold, since wall-clock timing +on shared CI runners is too noisy to gate on reliably). Run it locally before/after a change to +`iterate-loop.ts`, `iterate-policy.ts`, `attempt-metering.ts`, or `self-review-adapter.ts` to see whether +the change moved the needle under concurrency. + +## What it measures + +Each concurrency level runs a batch of simulated tenant attempts (a distinct `repoFullName`/ +`contributorLogin` per attempt, mirroring how independent tenants share the same AMS infra) through +`runIterateLoop`, `concurrency` attempts in flight at a time via `Promise.all`, until the configured +attempt count for that level completes. Every attempt is scripted to hand off after exactly one iteration +(a passing self-review verdict on the first try), so the wall-clock numbers isolate the loop's own +per-attempt overhead — driver invocation, self-review, policy decision, attempt-log append — from any +variation in how many iterations a real attempt would take. + +- **Concurrency levels:** 1, 8, 32, 128 concurrent attempts. +- **Attempts per level:** 32 (script default) / 64 (baseline capture below). +- **Simulated driver latency:** 15ms per iteration — a stand-in for the wall-clock a real coding-agent + subprocess invocation would take, without actually spending any real API budget or spawning a process. + +## Baseline (informational only, machine-dependent) + +Captured on a Linux x86_64 dev container, Node.js 22.23.1, 64 attempts per level. Absolute numbers vary by +hardware and by real driver/self-review latency — use this as a rough sense of scale and of how throughput +scales with concurrency, not a target: + +``` +iterate-loop load test + +concurrency=1: 1023.87ms wall for 64 attempts, 63 attempts/sec, 64/64 handed off (simulated driver latency 15ms) +concurrency=8: 151.74ms wall for 64 attempts, 422 attempts/sec, 64/64 handed off (simulated driver latency 15ms) +concurrency=32: 65.20ms wall for 64 attempts, 982 attempts/sec, 64/64 handed off (simulated driver latency 15ms) +concurrency=128: 40.59ms wall for 64 attempts, 1577 attempts/sec, 64/64 handed off (simulated driver latency 15ms) +``` + +Throughput scales roughly linearly with concurrency up to the point where the batch size matches (or +exceeds) the attempt count per level — at that point every attempt starts in the same tick and the +per-attempt overhead is fully parallelized, bounded only by the simulated driver latency plus the loop's own +synchronous work per attempt. This is execution/measurement only against the existing, already-injectable +driver seam; it does not change `runIterateLoop`'s own concurrency model. These numbers feed the per-tenant +scheduling and queue-fairness design work in the AMS Cloud Readiness milestone — reference them there rather +than re-measuring. diff --git a/packages/loopover-engine/scripts/load-test-iterate-loop.d.mts b/packages/loopover-engine/scripts/load-test-iterate-loop.d.mts new file mode 100644 index 0000000000..c67b0ea412 --- /dev/null +++ b/packages/loopover-engine/scripts/load-test-iterate-loop.d.mts @@ -0,0 +1,33 @@ +import type { CodingAgentDriver, CodingAgentDriverResult, CodingAgentDriverTask } from "../src/miner/coding-agent-driver.js"; + +export type LoadTestOptions = { + levels?: number[]; + attemptCount?: number; + latencyMs?: number; +}; + +export type LoadTestLevelResult = { + concurrency: number; + attemptCount: number; + latencyMs: number; + wallMs: number; + handoffCount: number; + attemptsPerSecond: number; +}; + +export declare const DEFAULT_CONCURRENCY_LEVELS: number[]; +export declare const DEFAULT_ATTEMPTS_PER_LEVEL: number; +export declare const DEFAULT_SIMULATED_DRIVER_LATENCY_MS: number; + +export declare function buildFakeLoadTestDriver(latencyMs: number): CodingAgentDriver & { + run(task: CodingAgentDriverTask): Promise; +}; + +export declare function runConcurrencyLevel( + concurrency: number, + options?: LoadTestOptions, +): Promise; + +export declare function runLoadTest(options?: LoadTestOptions): Promise; + +export declare function formatLoadTestReport(results: readonly LoadTestLevelResult[]): string; diff --git a/packages/loopover-engine/scripts/load-test-iterate-loop.mjs b/packages/loopover-engine/scripts/load-test-iterate-loop.mjs new file mode 100644 index 0000000000..395c6c78ed --- /dev/null +++ b/packages/loopover-engine/scripts/load-test-iterate-loop.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +import { performance } from "node:perf_hooks"; +import { parseFocusManifest, runIterateLoop } from "../dist/index.js"; + +// Load-testing harness for the AMS iterate-loop orchestrator (#5224): every existing iterate-loop test is +// correctness-oriented (single attempt, fake driver returns instantly), so there is no signal today for how +// runIterateLoop behaves when many tenants' attempts run concurrently against shared infra. This harness +// reuses the SAME fake-driver injection seam iterate-loop.test.ts already exercises (a CodingAgentDriver whose +// `run()` never spawns a real subprocess or spends API budget) but adds a configurable artificial per-iteration +// delay, so the measured throughput reflects iterate-loop's own orchestration/scheduling overhead under +// concurrency rather than a network call's latency. See docs/iterate-loop-load-test.md (#5224) for how to run +// this and read the numbers, and issue #4913 for the parallel Worker-endpoint load-testing precedent. + +export const DEFAULT_CONCURRENCY_LEVELS = [1, 8, 32, 128]; +export const DEFAULT_ATTEMPTS_PER_LEVEL = 32; +export const DEFAULT_SIMULATED_DRIVER_LATENCY_MS = 15; + +const SYNTHETIC_ISSUE_NUMBER = 7; + +/** One open issue per synthetic tenant repo, matching that tenant's own `passesPredictedGate` linkage below -- + * each tenant is a fully independent repo/contributor pair (`buildSelfReviewPredictedGateInput`'s own identity + * fields), the same "multi-tenant-like" shape the issue's Problem section asks this harness to load-test. */ +function buildReviewContext(tenantRepoFullName) { + return { + manifest: parseFocusManifest({ gate: { duplicates: "block", linkedIssue: "advisory" } }), + repo: { fullName: tenantRepoFullName, owner: tenantRepoFullName.split("/")[0], name: tenantRepoFullName.split("/")[1], isInstalled: true, isRegistered: true, isPrivate: false }, + issues: [{ repoFullName: tenantRepoFullName, number: SYNTHETIC_ISSUE_NUMBER, title: "Synthetic load-test issue", state: "open", labels: [], linkedPrs: [] }], + pullRequests: [], + }; +} + +/** A `CodingAgentDriver` (coding-agent-driver.ts) that never spawns a real subprocess or spends API budget -- + * it resolves after `latencyMs` (simulating the wall-clock an iteration of a real coding-agent invocation + * would take) with a scripted, always-passing result. `latencyMs` uses a real `setTimeout`, not a busy-loop, so + * concurrent attempts genuinely interleave on the event loop the way concurrent live attempts would. */ +export function buildFakeLoadTestDriver(latencyMs) { + return { + async run(task) { + await new Promise((resolve) => setTimeout(resolve, latencyMs)); + return { ok: true, changedFiles: [`src/${task.attemptId}.ts`], summary: `synthetic load-test change for ${task.attemptId}`, turnsUsed: 1 }; + }, + }; +} + +const NOOP_SLOP_ASSESSMENT = { slopRisk: 0, band: "clean", findings: [] }; + +/** One simulated tenant attempt: a distinct `repoFullName`/`contributorLogin` per `tenantIndex`, a linked open + * issue that matches on the first iteration, so every attempt hands off in exactly one iteration -- isolating + * the measurement to iterate-loop's own per-attempt orchestration overhead rather than varying iteration counts + * across runs. */ +async function runOneAttempt(tenantIndex, driver) { + const repoFullName = `load-test-tenant-${tenantIndex}/repo`; + const attemptId = `attempt-${tenantIndex}`; + const input = { + attemptId, + workingDirectory: `/tmp/${attemptId}`, + acceptanceCriteriaPath: `/tmp/${attemptId}/acceptance-criteria.json`, + instructions: "Synthetic load-test instructions", + mode: "live", + maxIterations: 3, + maxTurnsPerIteration: 20, + repoFullName, + contributorLogin: `miner-${tenantIndex}`, + title: "Synthetic load-test attempt", + body: `Closes #${SYNTHETIC_ISSUE_NUMBER}`, + linkedIssues: [SYNTHETIC_ISSUE_NUMBER], + reviewContext: buildReviewContext(repoFullName), + rejectionSignaled: false, + }; + const deps = { + driver, + runSlopAssessment: () => NOOP_SLOP_ASSESSMENT, + appendAttemptLogEvent: () => {}, + }; + const start = performance.now(); + const result = await runIterateLoop(input, deps); + return { elapsedMs: performance.now() - start, result }; +} + +/** + * Run `attemptCount` simulated tenant attempts concurrently (`Promise.all`, all started in the same tick) against + * one shared fake driver, and report the aggregate wall time plus derived throughput. Every attempt is expected + * to hand off after its first iteration (see {@link runOneAttempt}) -- a non-`"handoff"` outcome or a driver + * error would silently understate real concurrent load, so both are counted and surfaced rather than ignored. + */ +export async function runConcurrencyLevel(concurrency, options = {}) { + const attemptCount = options.attemptCount ?? DEFAULT_ATTEMPTS_PER_LEVEL; + const latencyMs = options.latencyMs ?? DEFAULT_SIMULATED_DRIVER_LATENCY_MS; + const driver = buildFakeLoadTestDriver(latencyMs); + + const start = performance.now(); + const outcomes = []; + for (let batchStart = 0; batchStart < attemptCount; batchStart += concurrency) { + const batchSize = Math.min(concurrency, attemptCount - batchStart); + const batch = await Promise.all( + Array.from({ length: batchSize }, (_unused, offset) => runOneAttempt(batchStart + offset, driver)), + ); + outcomes.push(...batch); + } + const wallMs = performance.now() - start; + const handoffCount = outcomes.filter((o) => o.result.outcome === "handoff").length; + + return { + concurrency, + attemptCount, + latencyMs, + wallMs, + handoffCount, + attemptsPerSecond: attemptCount / (wallMs / 1000), + }; +} + +/** Run every concurrency level in `levels` in sequence (never overlapping each other), so one level's + * scheduling contention never bleeds into the next level's measurement. */ +export async function runLoadTest(options = {}) { + const levels = options.levels ?? DEFAULT_CONCURRENCY_LEVELS; + const results = []; + for (const concurrency of levels) { + results.push(await runConcurrencyLevel(concurrency, options)); + } + return results; +} + +/** Render load-test results as a stable, greppable text report (no locale-dependent number formatting). */ +export function formatLoadTestReport(results) { + const lines = ["iterate-loop load test", ""]; + for (const r of results) { + lines.push( + `concurrency=${r.concurrency}: ${r.wallMs.toFixed(2)}ms wall for ${r.attemptCount} attempts, ` + + `${Math.round(r.attemptsPerSecond)} attempts/sec, ${r.handoffCount}/${r.attemptCount} handed off ` + + `(simulated driver latency ${r.latencyMs}ms)`, + ); + } + return lines.join("\n"); +} + +async function main() { + const results = await runLoadTest(); + console.log(formatLoadTestReport(results)); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/test/unit/iterate-loop-load-test-script.test.ts b/test/unit/iterate-loop-load-test-script.test.ts new file mode 100644 index 0000000000..88533171bd --- /dev/null +++ b/test/unit/iterate-loop-load-test-script.test.ts @@ -0,0 +1,89 @@ +import { performance } from "node:perf_hooks"; +import { spawnSync } from "node:child_process"; +import { describe, expect, it } from "vitest"; +import { + DEFAULT_ATTEMPTS_PER_LEVEL, + DEFAULT_CONCURRENCY_LEVELS, + DEFAULT_SIMULATED_DRIVER_LATENCY_MS, + buildFakeLoadTestDriver, + formatLoadTestReport, + runConcurrencyLevel, + runLoadTest, +} from "../../packages/loopover-engine/scripts/load-test-iterate-loop.mjs"; + +describe("iterate-loop load-test script (#5224)", () => { + it("the fake driver never spawns a real subprocess and resolves a scripted ok result after the configured latency", async () => { + const driver = buildFakeLoadTestDriver(5); + const start = performance.now(); + const result = await driver.run({ + attemptId: "attempt-0", + workingDirectory: "/tmp/attempt-0", + acceptanceCriteriaPath: "/tmp/attempt-0/acceptance-criteria.json", + instructions: "synthetic", + maxTurns: 1, + }); + expect(performance.now() - start).toBeGreaterThanOrEqual(4); + expect(result.ok).toBe(true); + expect(result.changedFiles).toEqual(["src/attempt-0.ts"]); + expect(result.turnsUsed).toBe(1); + }); + + it("runs a small concurrency level end-to-end and every attempt hands off on its first iteration", async () => { + const level = await runConcurrencyLevel(4, { attemptCount: 8, latencyMs: 1 }); + expect(level.concurrency).toBe(4); + expect(level.attemptCount).toBe(8); + expect(level.latencyMs).toBe(1); + expect(level.handoffCount).toBe(8); + expect(Number.isFinite(level.wallMs)).toBe(true); + expect(level.wallMs).toBeGreaterThan(0); + expect(Number.isFinite(level.attemptsPerSecond)).toBe(true); + expect(level.attemptsPerSecond).toBeGreaterThan(0); + }); + + it("runs a concurrency level where the batch size exceeds the attempt count in a single batch", async () => { + const level = await runConcurrencyLevel(128, { attemptCount: 3, latencyMs: 1 }); + expect(level.attemptCount).toBe(3); + expect(level.handoffCount).toBe(3); + }); + + it("runs every concurrency level supplied via options.levels, in order", async () => { + const results = await runLoadTest({ levels: [1, 2], attemptCount: 2, latencyMs: 1 }); + expect(results).toHaveLength(2); + expect(results.map((r) => r.concurrency)).toEqual([1, 2]); + for (const r of results) expect(r.handoffCount).toBe(2); + }); + + it("exposes the documented default concurrency levels, attempt count, and simulated latency", () => { + expect(DEFAULT_CONCURRENCY_LEVELS).toEqual([1, 8, 32, 128]); + expect(DEFAULT_ATTEMPTS_PER_LEVEL).toBe(32); + expect(DEFAULT_SIMULATED_DRIVER_LATENCY_MS).toBe(15); + }); + + it("renders a deterministic report with no locale-dependent number formatting", () => { + expect( + formatLoadTestReport([ + { concurrency: 1, attemptCount: 10, latencyMs: 15, wallMs: 160.4, handoffCount: 10, attemptsPerSecond: 62.34 }, + { concurrency: 8, attemptCount: 10, latencyMs: 15, wallMs: 20.1, handoffCount: 9, attemptsPerSecond: 497.5 }, + ]), + ).toBe( + [ + "iterate-loop load test", + "", + "concurrency=1: 160.40ms wall for 10 attempts, 62 attempts/sec, 10/10 handed off (simulated driver latency 15ms)", + "concurrency=8: 20.10ms wall for 10 attempts, 498 attempts/sec, 9/10 handed off (simulated driver latency 15ms)", + ].join("\n"), + ); + }); + + it("runs end-to-end as a CLI script and prints the report header plus every default concurrency level", () => { + const result = spawnSync(process.execPath, ["packages/loopover-engine/scripts/load-test-iterate-loop.mjs"], { + cwd: process.cwd(), + encoding: "utf8", + }); + expect(result.status).toBe(0); + expect(result.stdout).toContain("iterate-loop load test"); + for (const level of DEFAULT_CONCURRENCY_LEVELS) { + expect(result.stdout).toContain(`concurrency=${level}:`); + } + }); +});