diff --git a/packages/gittensory-engine/src/fleet-run-manifest.ts b/packages/gittensory-engine/src/fleet-run-manifest.ts new file mode 100644 index 0000000000..04c2e78ec7 --- /dev/null +++ b/packages/gittensory-engine/src/fleet-run-manifest.ts @@ -0,0 +1,191 @@ +import { parse as parseYaml } from "yaml"; + +// FleetRunManifest (#4299). The top-level config a *fleet operator* authors to run the miner across many repos: +// which repos are in scope for a fleet run, and how a finite worktree/concurrency budget is split between them. +// This is the OPERATOR-side analogue of, and deliberately NOT the same file as, `.gittensory-miner.yml` (see +// miner-goal-spec.ts): that one is authored by a *target repo's maintainer* to say how their one repo wants to be +// approached. Same tolerant-parser convention (every field optional, unknown keys ignored, malformed input +// degrades to a documented default with a warning rather than throwing); opposite author and direction of intent. +// See packages/gittensory-miner/docs/fleet-run-manifest.md for the full distinction. + +/** One target repo in a fleet run, with its own concurrent-worktree budget. */ +export type FleetRunManifestRepo = { + /** Canonical `owner/repo`. Compatible with opportunity-fanout's target normalization (a splittable pair). */ + repoFullName: string; + /** + * Max concurrent worktrees (in-flight attempts) this repo may hold at once. A positive integer (`>= 1`); a + * non-integer is floored, a value below 1 falls back to the default. Default: 1. + */ + maxConcurrentWorktrees: number; +}; + +/** Fleet run-manifest: the repos to work across and how to split the concurrency budget. See {@link DEFAULT_FLEET_RUN_MANIFEST}. */ +export type FleetRunManifest = { + /** Target repos, de-duplicated by `repoFullName` (first entry wins). Default: [] (no repos in scope). */ + repos: readonly FleetRunManifestRepo[]; + /** + * Total concurrent worktrees across the whole fleet, regardless of per-repo budgets. A positive integer + * (`>= 1`); floored, sub-1 falls back to the default. Default: 1. + */ + totalConcurrentWorktrees: number; +}; + +/** Tolerant parser result: the normalized manifest plus warnings and whether the file expressed any non-default + * field. Mirrors {@link ParsedMinerGoalSpec}'s present/warnings shape. */ +export type ParsedFleetRunManifest = { + present: boolean; + manifest: FleetRunManifest; + warnings: string[]; +}; + +/** Safe defaults applied when a field is absent (or the file is missing): no repos in scope, one worktree total. + * Deep-frozen shared singleton — clone before layering overrides. */ +export const DEFAULT_FLEET_RUN_MANIFEST: FleetRunManifest = Object.freeze({ + repos: Object.freeze([]), + totalConcurrentWorktrees: 1, +}); + +const MAX_FLEET_RUN_MANIFEST_BYTES = 65_536; +const MAX_MANIFEST_REPOS = 500; + +function cloneDefaultFleetRunManifest(): FleetRunManifest { + return { ...DEFAULT_FLEET_RUN_MANIFEST, repos: [...DEFAULT_FLEET_RUN_MANIFEST.repos] }; +} + +function emptyFleetRunManifest(warnings: string[] = []): ParsedFleetRunManifest { + return { present: false, manifest: cloneDefaultFleetRunManifest(), warnings }; +} + +/** `owner/repo` with exactly one slash and non-empty halves; anything else → null. Same shape the goal-spec / + * portfolio-queue validators use, so a manifest repo is directly compatible with opportunity-fanout targets. */ +function normalizeRepoFullName(value: unknown): string | null { + if (typeof value !== "string") return null; + const [owner, repo, extra] = value.trim().split("/"); + if (!owner || !repo || extra !== undefined) return null; + return `${owner}/${repo}`; +} + +function normalizePositiveInteger(value: unknown, field: string, fallback: number, warnings: string[]): number { + if (value === undefined || value === null) return fallback; + if (typeof value !== "number" || !Number.isFinite(value)) { + warnings.push(`FleetRunManifest field "${field}" must be a positive whole number; falling back to ${fallback}.`); + return fallback; + } + const normalized = Math.floor(value); + if (normalized >= 1) return normalized; + warnings.push(`FleetRunManifest field "${field}" must be >= 1 after flooring; falling back to ${fallback}.`); + return fallback; +} + +// A repo entry may be a bare `"owner/repo"` string (uses the default per-repo budget) or a `{ repoFullName, +// maxConcurrentWorktrees? }` mapping. Anything else, or an unparseable repo name, is skipped with a warning. +function normalizeRepoList(value: unknown, warnings: string[]): FleetRunManifestRepo[] { + if (value === undefined || value === null) return []; + if (!Array.isArray(value)) { + warnings.push(`FleetRunManifest field "repos" must be a list; ignoring a ${typeof value} value.`); + return []; + } + const result: FleetRunManifestRepo[] = []; + const seen = new Set(); + for (const [index, entry] of value.entries()) { + if (index >= MAX_MANIFEST_REPOS) { + warnings.push(`FleetRunManifest field "repos" exceeded ${MAX_MANIFEST_REPOS} entries; extra entries ignored.`); + break; + } + let repoFullName: string | null; + let maxConcurrentWorktrees = DEFAULT_FLEET_RUN_MANIFEST.totalConcurrentWorktrees; + if (typeof entry === "string") { + repoFullName = normalizeRepoFullName(entry); + } else if (entry && typeof entry === "object" && !Array.isArray(entry)) { + const record = entry as Record; + repoFullName = normalizeRepoFullName(record.repoFullName); + maxConcurrentWorktrees = normalizePositiveInteger(record.maxConcurrentWorktrees, "maxConcurrentWorktrees", 1, warnings); + } else { + warnings.push(`FleetRunManifest "repos" skipped a non-string, non-mapping entry.`); + continue; + } + if (repoFullName === null) { + warnings.push(`FleetRunManifest "repos" skipped an entry with an invalid "owner/repo" name.`); + continue; + } + if (seen.has(repoFullName)) { + warnings.push(`FleetRunManifest "repos" skipped a duplicate entry for ${repoFullName}.`); + continue; + } + seen.add(repoFullName); + result.push({ repoFullName, maxConcurrentWorktrees }); + } + return result; +} + +function utf8ByteLength(value: string): number { + let bytes = 0; + for (const char of value) { + const codePoint = char.codePointAt(0) as number; + if (codePoint <= 0x7f) bytes += 1; + else if (codePoint <= 0x7ff) bytes += 2; + else if (codePoint <= 0xffff) bytes += 3; + else bytes += 4; + } + return bytes; +} + +function hasConfiguredManifestFields(manifest: FleetRunManifest): boolean { + return manifest.repos.length > 0 || manifest.totalConcurrentWorktrees !== DEFAULT_FLEET_RUN_MANIFEST.totalConcurrentWorktrees; +} + +/** + * Tolerantly normalize an already-parsed run-manifest object into a {@link ParsedFleetRunManifest}. Never throws: + * malformed shapes degrade to safe defaults and accumulate warnings so a fleet run can surface "your run-manifest + * had problems" without hard-failing. Mirrors {@link parseMinerGoalSpec}. + */ +export function parseFleetRunManifest(raw: unknown): ParsedFleetRunManifest { + if (raw === undefined || raw === null) return emptyFleetRunManifest(); + if (typeof raw !== "object" || Array.isArray(raw)) { + return emptyFleetRunManifest([ + "FleetRunManifest must be a mapping of fields; ignoring malformed config and falling back to safe defaults.", + ]); + } + const record = raw as Record; + const warnings: string[] = []; + const manifest: FleetRunManifest = { + repos: normalizeRepoList(record.repos, warnings), + totalConcurrentWorktrees: normalizePositiveInteger( + record.totalConcurrentWorktrees, + "totalConcurrentWorktrees", + DEFAULT_FLEET_RUN_MANIFEST.totalConcurrentWorktrees, + warnings, + ), + }; + if (!hasConfiguredManifestFields(manifest)) { + warnings.push("FleetRunManifest contained no recognized non-default fields; falling back to safe defaults."); + return { present: false, manifest: cloneDefaultFleetRunManifest(), warnings }; + } + return { present: true, manifest, warnings }; +} + +/** + * Parse raw run-manifest file content (JSON or YAML). Malformed content degrades to an absent manifest with a + * warning rather than throwing, mirroring {@link parseMinerGoalSpecContent}. + */ +export function parseFleetRunManifestContent(content: string | null | undefined): ParsedFleetRunManifest { + if (content === undefined || content === null || content.trim() === "") return emptyFleetRunManifest(); + if (utf8ByteLength(content) > MAX_FLEET_RUN_MANIFEST_BYTES) { + return emptyFleetRunManifest([ + `FleetRunManifest content exceeded ${MAX_FLEET_RUN_MANIFEST_BYTES} bytes; ignoring it and falling back to safe defaults.`, + ]); + } + const trimmed = content.trim(); + const looksLikeJson = trimmed.startsWith("{") || trimmed.startsWith("["); + let parsed: unknown; + try { + parsed = looksLikeJson ? JSON.parse(trimmed) : parseYaml(trimmed); + } catch { + return emptyFleetRunManifest([ + looksLikeJson + ? "FleetRunManifest content was not valid JSON; ignoring it and falling back to safe defaults." + : "FleetRunManifest content was not valid YAML; ignoring it and falling back to safe defaults.", + ]); + } + return parseFleetRunManifest(parsed); +} diff --git a/packages/gittensory-engine/src/index.ts b/packages/gittensory-engine/src/index.ts index 617d471a1c..28adfb66e1 100644 --- a/packages/gittensory-engine/src/index.ts +++ b/packages/gittensory-engine/src/index.ts @@ -197,6 +197,14 @@ export { type MinerIssueDiscoveryPolicy, type ParsedMinerGoalSpec, } from "./miner-goal-spec.js"; +export { + DEFAULT_FLEET_RUN_MANIFEST, + parseFleetRunManifest, + parseFleetRunManifestContent, + type FleetRunManifest, + type FleetRunManifestRepo, + type ParsedFleetRunManifest, +} from "./fleet-run-manifest.js"; export { computeMetadataLaneFit, computeMinerGoalLaneFit, diff --git a/packages/gittensory-miner/docs/fleet-run-manifest.md b/packages/gittensory-miner/docs/fleet-run-manifest.md new file mode 100644 index 0000000000..cf2b371a1a --- /dev/null +++ b/packages/gittensory-miner/docs/fleet-run-manifest.md @@ -0,0 +1,50 @@ +# Fleet run-manifest + +The **fleet run-manifest** is the top-level config a *fleet operator* authors to run the miner across many repos +at once: it declares which repos are in scope for a fleet run and how a finite worktree/concurrency budget is +split between them. It is parsed by `parseFleetRunManifestContent` / `parseFleetRunManifest` in +`@jsonbored/gittensory-engine` (`packages/gittensory-engine/src/fleet-run-manifest.ts`). + +It is **not** the same file as `.gittensory-miner.yml` (see [`miner-goal-spec.md`](./miner-goal-spec.md)) — the +naming is easy to conflate. Same tolerant-parser convention (every field optional, unknown keys ignored, a +malformed field degrades to a documented default with a warning rather than throwing), but the opposite author +and the opposite direction of intent: + +| | `.gittensory-miner.yml` (goal spec) | fleet run-manifest | +|---|---|---| +| **Author** | a target repo's maintainer | the miner (fleet) operator | +| **Lives in** | the target repo | the operator's fleet-run config | +| **Direction** | how this one repo wants to be approached | which repos to work across, and how to split the budget | +| **Scope** | one repo | many repos in one run | +| **Key fields** | `minerEnabled`, `wantedPaths`, `blockedPaths`, `preferredLabels`, `blockedLabels`, `maxConcurrentClaims`, `issueDiscoveryPolicy` | `repos` (each `owner/repo` + `maxConcurrentWorktrees`), `totalConcurrentWorktrees` | + +## Schema + +Every field is optional; unknown keys are ignored; a malformed field falls back to a documented default with a +warning rather than hard-failing the run. + +- **`repos`** — a list of target repos. Each entry is either a bare `"owner/repo"` string (uses the default + per-repo budget) or a `{ repoFullName, maxConcurrentWorktrees }` mapping. Invalid or duplicate entries are + skipped with a warning. `repoFullName` is a canonical `owner/repo`, compatible with `opportunity-fanout.js`'s + target list. Default: `[]`. +- **`repos[].maxConcurrentWorktrees`** — max concurrent worktrees (in-flight attempts) for that repo. A positive + integer (floored; sub-1 falls back to the default). Default: `1`. +- **`totalConcurrentWorktrees`** — total concurrent worktrees across the whole fleet, regardless of per-repo + budgets. A positive integer. Default: `1`. + +## Wiring + +This module produces only the parsed, typed manifest. Driving the fleet concurrency allocator from it is the +allocator's concern (sibling `feat(miner-concurrency): add git-worktree-per-attempt allocator` issue), and the +cross-repo `portfolio-queue.js` backlog reads the same repo list — both *consume* this manifest; neither wiring +lives here. + +## Example (`fleet-run.yml`) + +```yaml +totalConcurrentWorktrees: 4 +repos: + - owner/repo-a + - repoFullName: owner/repo-b + maxConcurrentWorktrees: 2 +``` diff --git a/test/unit/fleet-run-manifest-parser.test.ts b/test/unit/fleet-run-manifest-parser.test.ts new file mode 100644 index 0000000000..667e1d68d0 --- /dev/null +++ b/test/unit/fleet-run-manifest-parser.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_FLEET_RUN_MANIFEST, + parseFleetRunManifest, + parseFleetRunManifestContent, +} from "../../packages/gittensory-engine/src/index"; + +describe("FleetRunManifest parser (#4299)", () => { + it("re-exports the parser API from the engine barrel", () => { + expect(typeof parseFleetRunManifest).toBe("function"); + expect(typeof parseFleetRunManifestContent).toBe("function"); + }); + + it("treats missing raw input as an absent safe-default manifest", () => { + for (const raw of [undefined, null]) { + expect(parseFleetRunManifest(raw)).toEqual({ present: false, manifest: DEFAULT_FLEET_RUN_MANIFEST, warnings: [] }); + } + }); + + it.each(["not a mapping", ["still", "not", "a", "mapping"]])("degrades a malformed top-level value to safe defaults: %j", (raw) => { + const parsed = parseFleetRunManifest(raw); + expect(parsed.present).toBe(false); + expect(parsed.manifest).toEqual(DEFAULT_FLEET_RUN_MANIFEST); + expect(parsed.warnings.join(" ")).toMatch(/must be a mapping/i); + }); + + it("treats an all-default mapping as absent (no non-default fields)", () => { + const parsed = parseFleetRunManifest({ repos: [], totalConcurrentWorktrees: 1 }); + expect(parsed.present).toBe(false); + expect(parsed.warnings.join(" ")).toMatch(/no recognized non-default fields/i); + }); + + it("normalizes string + object repo entries, floors budgets, dedupes, and skips invalid entries", () => { + const parsed = parseFleetRunManifest({ + repos: [ + "owner/a", // string → default budget 1 + { repoFullName: "owner/b", maxConcurrentWorktrees: 3.9 }, // object → floored to 3 + { repoFullName: "owner/b", maxConcurrentWorktrees: 2 }, // duplicate → skipped + "owner/a", // duplicate → skipped + "not-a-repo", // invalid name (no slash) → skipped + "owner/repo/extra", // invalid name (too many slashes) → skipped + "/only-repo", // invalid name (empty owner) → skipped + { repoFullName: "no-slash" }, // invalid name → skipped + { repoFullName: 123 }, // non-string repoFullName → skipped + { repoFullName: "owner/c", maxConcurrentWorktrees: "x" }, // non-numeric budget → default 1 + warning + 42, // non-string / non-mapping → skipped + ], + totalConcurrentWorktrees: 5, + }); + expect(parsed.present).toBe(true); + expect(parsed.manifest.repos).toEqual([ + { repoFullName: "owner/a", maxConcurrentWorktrees: 1 }, + { repoFullName: "owner/b", maxConcurrentWorktrees: 3 }, + { repoFullName: "owner/c", maxConcurrentWorktrees: 1 }, + ]); + expect(parsed.manifest.totalConcurrentWorktrees).toBe(5); + const w = parsed.warnings.join(" "); + expect(w).toMatch(/duplicate entry for owner\/b/); + expect(w).toMatch(/invalid "owner\/repo" name/); + expect(w).toMatch(/non-string, non-mapping/); + expect(w).toMatch(/"maxConcurrentWorktrees" must be a positive whole number/); + }); + + it("falls a non-list repos field and a sub-1 total budget back to defaults with warnings", () => { + const parsed = parseFleetRunManifest({ repos: "owner/a", totalConcurrentWorktrees: 0 }); + expect(parsed.manifest.repos).toEqual([]); + expect(parsed.manifest.totalConcurrentWorktrees).toBe(1); + const w = parsed.warnings.join(" "); + expect(w).toMatch(/"repos" must be a list/); + expect(w).toMatch(/"totalConcurrentWorktrees" must be >= 1/); + }); + + it("warns on a non-numeric total budget", () => { + const parsed = parseFleetRunManifest({ repos: ["owner/a"], totalConcurrentWorktrees: "lots" }); + expect(parsed.present).toBe(true); + expect(parsed.manifest.totalConcurrentWorktrees).toBe(1); + expect(parsed.warnings.join(" ")).toMatch(/"totalConcurrentWorktrees" must be a positive whole number/); + }); + + it("caps the repo list and warns when it is exceeded", () => { + const many = Array.from({ length: 502 }, (_, i) => `owner/r${i}`); + const parsed = parseFleetRunManifest({ repos: many }); + expect(parsed.manifest.repos).toHaveLength(500); + expect(parsed.warnings.join(" ")).toMatch(/exceeded 500 entries/); + }); + + it("parseFleetRunManifestContent: blank / missing content is an absent manifest", () => { + for (const content of [undefined, null, "", " "]) { + expect(parseFleetRunManifestContent(content)).toEqual({ present: false, manifest: DEFAULT_FLEET_RUN_MANIFEST, warnings: [] }); + } + }); + + it("parseFleetRunManifestContent: parses YAML and JSON, over-limit + malformed degrade with a warning", () => { + const yaml = parseFleetRunManifestContent("repos:\n - owner/a\n - repoFullName: owner/b\n maxConcurrentWorktrees: 2\ntotalConcurrentWorktrees: 4\n"); + expect(yaml.present).toBe(true); + expect(yaml.manifest.repos.map((r) => r.repoFullName)).toEqual(["owner/a", "owner/b"]); + expect(yaml.manifest.totalConcurrentWorktrees).toBe(4); + + const json = parseFleetRunManifestContent('{"repos":["owner/a"],"totalConcurrentWorktrees":3}'); + expect(json.present).toBe(true); + expect(json.manifest.totalConcurrentWorktrees).toBe(3); + + // multi-byte content (still under the byte limit) exercises the byte-length accounting. + expect(parseFleetRunManifestContent("totalConcurrentWorktrees: 2 # é中\u{1F600}").manifest.totalConcurrentWorktrees).toBe(2); + + expect(parseFleetRunManifestContent('{"repos": [invalid json}').warnings.join(" ")).toMatch(/not valid JSON/); + expect(parseFleetRunManifestContent("repos:\n - : :\n :bad").warnings.join(" ")).toMatch(/not valid YAML/); + expect(parseFleetRunManifestContent("x".repeat(65_537)).warnings.join(" ")).toMatch(/exceeded 65536 bytes/); + }); +});