From f21c43f06d140db1c4ded89f2eda890a92b67014 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:48:53 -0700 Subject: [PATCH] feat(tooling): feed recorded drafting misses back into every issue draft as a pre-publish checklist (#8118) Every draft was independent -- the tool had no memory of past post-merge gaps, the exact failure mode the maintainer named as the bottleneck. Now a real gap, once found, is recorded manually via scripts/record-drafting-miss.ts (appends to scripts/drafting-misses.json -- a plain committed file, no new database) and draftIssueBody applies the accumulated lessons on EVERY subsequent draft: a checklist section grouped per category (repeats counted, most recent lesson wording wins), under the same resolve-then-DELETE contract as the UNGROUNDED markers so it can never survive publishing. parseDraftingMisses is deliberately fail-loud -- silently dropping a malformed lesson would defeat the loop -- and the recorder re-validates the whole file through the same parser on every append so a broken hand-edit surfaces at record time, not on the next draft. Still zero auto-detection: a human decides what counts as a miss (#8118's own boundary). 100% line+branch coverage on the extended core. --- scripts/draft-issue.ts | 43 +++++++++++++---- scripts/record-drafting-miss.ts | 56 ++++++++++++++++++++++ src/services/issue-drafting.ts | 81 ++++++++++++++++++++++++++++++++ test/unit/issue-drafting.test.ts | 71 ++++++++++++++++++++++++++++ 4 files changed, 243 insertions(+), 8 deletions(-) create mode 100644 scripts/record-drafting-miss.ts diff --git a/scripts/draft-issue.ts b/scripts/draft-issue.ts index 9a2feb3242..0dad192d1e 100644 --- a/scripts/draft-issue.ts +++ b/scripts/draft-issue.ts @@ -7,17 +7,29 @@ // export-d1-core.ts. NEVER publishes anything, NEVER touches labels/milestones — see the core's own // boundary comment. // -// tsx scripts/draft-issue.ts --prompt "" --output [--root .] -// tsx scripts/draft-issue.ts --prompt-file --output [--root .] -import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +// tsx scripts/draft-issue.ts --prompt "" --output [--root .] [--misses ] +// tsx scripts/draft-issue.ts --prompt-file --output [--root .] [--misses ] +// +// #8118: every draft automatically applies the accumulated drafting-miss lessons from +// scripts/drafting-misses.json (recorded via scripts/record-drafting-miss.ts) when that file exists; +// --misses points at a different file. A malformed misses file fails the draft loudly — see +// parseDraftingMisses's own fail-loud rationale. +import { existsSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { join, relative } from "node:path"; -import { draftIssueBody, type CorpusFile } from "../src/services/issue-drafting.js"; +import { + DEFAULT_DRAFTING_MISSES_FILE, + draftIssueBody, + parseDraftingMisses, + type CorpusFile, + type DraftingMiss, +} from "../src/services/issue-drafting.js"; type Args = { prompt: string | undefined; promptFile: string | undefined; output: string | undefined; root: string; + misses: string | undefined; }; // The corpus mirrors where real precedent lives (the gate's own wantedPaths, minus content-free dirs). @@ -27,17 +39,30 @@ const SKIP_DIR_NAMES = new Set(["node_modules", "dist", "coverage", ".turbo"]); const MAX_FILE_BYTES = 512 * 1024; function parseArgs(argv: string[]): Args { - const args: Args = { prompt: undefined, promptFile: undefined, output: undefined, root: "." }; + const args: Args = { prompt: undefined, promptFile: undefined, output: undefined, root: ".", misses: undefined }; for (let i = 0; i < argv.length; i += 1) { const flag = argv[i]; if (flag === "--prompt") args.prompt = argv[++i]; else if (flag === "--prompt-file") args.promptFile = argv[++i]; else if (flag === "--output") args.output = argv[++i]; else if (flag === "--root") args.root = argv[++i]!; + else if (flag === "--misses") args.misses = argv[++i]; } return args; } +// #8118: apply the accumulated misses on EVERY draft — the default file is picked up automatically when it +// exists, so the loop needs no flag to keep working; an explicitly-passed path must exist (a typo silently +// drafting without the checklist would defeat the loop). +function loadDraftingMisses(root: string, explicitPath: string | undefined): DraftingMiss[] { + const path = explicitPath ?? join(root, DEFAULT_DRAFTING_MISSES_FILE); + if (!existsSync(path)) { + if (explicitPath) throw new Error(`--misses file not found: ${explicitPath}`); + return []; + } + return parseDraftingMisses(readFileSync(path, "utf8")); +} + function collectCorpus(root: string): CorpusFile[] { const corpus: CorpusFile[] = []; const walk = (dir: string) => { @@ -65,16 +90,18 @@ function main() { const args = parseArgs(process.argv.slice(2)); const prompt = args.prompt ?? (args.promptFile ? readFileSync(args.promptFile, "utf8") : undefined); if (!prompt || !args.output) { - console.error("Usage: tsx scripts/draft-issue.ts (--prompt | --prompt-file ) --output [--root .]"); + console.error("Usage: tsx scripts/draft-issue.ts (--prompt | --prompt-file ) --output [--root .] [--misses ]"); process.exit(2); } const corpus = collectCorpus(args.root); - const result = draftIssueBody(prompt, corpus); + const misses = loadDraftingMisses(args.root, args.misses); + const result = draftIssueBody(prompt, corpus, { misses }); writeFileSync(args.output, result.body); console.error( `drafted from ${corpus.length} corpus file(s): ${result.groundedTerms.length} term(s) grounded, ` + - `${result.ungroundedTerms.length} UNGROUNDED marker(s) to resolve by hand → ${args.output}`, + `${result.ungroundedTerms.length} UNGROUNDED marker(s) to resolve by hand, ` + + `${misses.length} recorded miss(es) applied → ${args.output}`, ); console.error("review + edit before publishing — this tool never publishes, and labels/milestone stay your call."); } diff --git a/scripts/record-drafting-miss.ts b/scripts/record-drafting-miss.ts new file mode 100644 index 0000000000..1b3e550dcc --- /dev/null +++ b/scripts/record-drafting-miss.ts @@ -0,0 +1,56 @@ +#!/usr/bin/env node +// Drafting-miss recorder (#8118, extends #8103) — the cheap, MANUAL way the maintainer flags a real +// post-merge gap traceable to a drafted issue: something the draft should have specified but didn't. +// Appends one validated record to the shared misses file (scripts/drafting-misses.json by default); +// scripts/draft-issue.ts reads that file on every subsequent draft and renders the accumulated lessons as +// a pre-publish checklist. Nothing here auto-detects gaps — a human decides what counts as a miss, this +// just captures it once found. Thin IO wrapper; the validation lives in the core's parseDraftingMisses. +// +// tsx scripts/record-drafting-miss.ts --prompt "" --missing "" \ +// [--category ] [--file scripts/drafting-misses.json] +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { DEFAULT_DRAFTING_MISSES_FILE, parseDraftingMisses, type DraftingMiss } from "../src/services/issue-drafting.js"; + +type Args = { + prompt: string | undefined; + missing: string | undefined; + category: string | undefined; + file: string; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { prompt: undefined, missing: undefined, category: undefined, file: DEFAULT_DRAFTING_MISSES_FILE }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--prompt") args.prompt = argv[++i]; + else if (flag === "--missing") args.missing = argv[++i]; + else if (flag === "--category") args.category = argv[++i]; + else if (flag === "--file") args.file = argv[++i]!; + } + return args; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args.prompt || !args.missing) { + console.error( + 'Usage: tsx scripts/record-drafting-miss.ts --prompt "" --missing "" [--category ] [--file scripts/drafting-misses.json]', + ); + process.exit(2); + } + + // Re-validate the whole file through the core parser on every append, so a hand-edit that broke it is + // caught here (at record time) instead of failing the next draft. + const existing: DraftingMiss[] = existsSync(args.file) ? parseDraftingMisses(readFileSync(args.file, "utf8")) : []; + const miss: DraftingMiss = { + recordedAt: new Date().toISOString(), + loosePrompt: args.prompt, + missing: args.missing, + ...(args.category ? { category: args.category } : {}), + }; + existing.push(miss); + writeFileSync(args.file, `${JSON.stringify(existing, null, 2)}\n`); + console.error(`recorded drafting miss #${existing.length}${args.category ? ` [${args.category}]` : ""} → ${args.file}`); +} + +main(); diff --git a/src/services/issue-drafting.ts b/src/services/issue-drafting.ts index 745297d164..0c301385fc 100644 --- a/src/services/issue-drafting.ts +++ b/src/services/issue-drafting.ts @@ -33,11 +33,33 @@ export type GroundingMatch = { path: string; line: number; text: string }; /** A term from the loose prompt that WAS grounded in real precedent. */ export type GroundedTerm = GroundingTerm & { matches: readonly GroundingMatch[] }; +/** One recorded drafting miss (#8118): a real post-merge gap traceable to a draft that should have + * specified something and didn't. Recorded manually by the maintainer after the fact — this module never + * auto-detects gaps, it only learns from the ones a human already confirmed. */ +export type DraftingMiss = { + /** ISO timestamp of when the miss was recorded. */ + recordedAt: string; + /** The loose prompt the flawed draft was generated from. */ + loosePrompt: string; + /** What the draft should have specified but didn't — written as a reusable lesson. */ + missing: string; + /** Optional gap category ("unstated-anti-pattern", "unverified-signature", …) used to dedupe the + * checklist: two misses in one category render as one checklist line with a ×N count. */ + category?: string; +}; + +/** Repo-relative default location of the misses file, shared by the recorder and drafter CLIs (#8118) — + * a plain committed JSON file per the issue's "no new database" boundary. The core only exports the + * string; reading/writing it stays the thin consumers' IO. */ +export const DEFAULT_DRAFTING_MISSES_FILE = "scripts/drafting-misses.json"; + export type IssueDraftOptions = { /** Cap on distinct terms extracted from the prompt (default 12). */ maxTerms?: number; /** Cap on matches kept per grounded term (default 3). */ maxMatchesPerTerm?: number; + /** Accumulated drafting misses (#8118) — rendered into every draft as a pre-publish checklist. */ + misses?: readonly DraftingMiss[]; }; export type IssueDraftResult = { @@ -146,6 +168,54 @@ export function groundTerm( return { ...groundingTerm, matches: matches.slice(0, Math.max(1, maxMatchesPerTerm)) }; } +/** + * Parse the drafting-misses file's JSON content (#8118) into validated {@link DraftingMiss} records. + * FAIL-LOUD, deliberately: this is the maintainer's own accumulated learning data, and silently dropping a + * malformed lesson would defeat the entire feedback loop — a broken file should stop the draft, not shrink + * the checklist. (Contrast with the corpus parsers' fail-open posture, which protect a live review pass.) + */ +export function parseDraftingMisses(json: string): DraftingMiss[] { + let parsed: unknown; + try { + parsed = JSON.parse(json); + } catch { + throw new Error("drafting-misses file is not valid JSON"); + } + if (!Array.isArray(parsed)) throw new Error("drafting-misses file must be a JSON array of miss records"); + return parsed.map((entry, index) => { + const record = (entry ?? {}) as Record; + if (typeof record.recordedAt !== "string" || !record.recordedAt || typeof record.loosePrompt !== "string" || typeof record.missing !== "string" || !record.missing) { + throw new Error(`drafting miss #${index} is malformed — need recordedAt, loosePrompt, and a non-empty missing lesson`); + } + const miss: DraftingMiss = { recordedAt: record.recordedAt, loosePrompt: record.loosePrompt, missing: record.missing }; + if (typeof record.category === "string" && record.category) miss.category = record.category; + return miss; + }); +} + +/** Collapse recorded misses into checklist lines: one line per category (uncategorized misses stay + * one-per-lesson), counting repeats and keeping the most recently recorded lesson text as the actionable + * wording. Sorted by label for byte-stable drafts. */ +function groupDraftingMisses(misses: readonly DraftingMiss[]): Array<{ label: string; count: number; lesson: string }> { + const groups = new Map(); + for (const miss of misses) { + const key = miss.category ?? `uncategorized:${miss.missing}`; + const existing = groups.get(key); + if (!existing) { + groups.set(key, { label: miss.category ?? "one-off", count: 1, lesson: miss.missing, lessonAt: miss.recordedAt }); + } else { + existing.count += 1; + if (miss.recordedAt > existing.lessonAt) { + existing.lesson = miss.missing; + existing.lessonAt = miss.recordedAt; + } + } + } + return [...groups.entries()] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([, group]) => ({ label: group.label, count: group.count, lesson: group.lesson })); +} + /** True when a cited path is graded by Codecov's patch gate (coverage.include: `src/**` and the engine's * `src/**` — mirrors codecov.yml's ignore list + vitest.config.ts's include, kept in sync by hand). */ function pathIsCoverageGraded(path: string): boolean { @@ -264,6 +334,17 @@ export function draftIssueBody(prompt: string, corpus: readonly CorpusFile[], op ); for (const path of citedPaths) lines.push(`- \`${path}\``); if (citedPaths.length === 0) lines.push("- "); + + // #8118: the accumulated-misses checklist — every recorded post-merge gap becomes a concrete + // double-check on every subsequent draft, so the tool gets better instead of repeating its misses. + // Same "resolve, then delete" contract as the UNGROUNDED markers: it must never survive publishing. + const misses = options.misses ?? []; + if (misses.length > 0) { + lines.push("", "## Pre-publish checklist — learned from recorded drafting misses. Resolve each item, then DELETE this section before publishing.", ""); + for (const group of groupDraftingMisses(misses)) { + lines.push(`- [ ] ${group.label}${group.count > 1 ? ` (recorded ${group.count}×)` : ""}: ${group.lesson}`); + } + } lines.push(""); return { body: lines.join("\n"), groundedTerms, ungroundedTerms }; diff --git a/test/unit/issue-drafting.test.ts b/test/unit/issue-drafting.test.ts index a804b611d2..1f0adffc48 100644 --- a/test/unit/issue-drafting.test.ts +++ b/test/unit/issue-drafting.test.ts @@ -1,9 +1,12 @@ import { describe, expect, it } from "vitest"; import { + DEFAULT_DRAFTING_MISSES_FILE, draftIssueBody, extractGroundingTerms, groundTerm, + parseDraftingMisses, type CorpusFile, + type DraftingMiss, type GroundingTerm, } from "../../src/services/issue-drafting"; @@ -236,3 +239,71 @@ describe("issue-drafting draftIssueBody (#8103)", () => { } }); }); + +describe("issue-drafting parseDraftingMisses (#8118)", () => { + const validMiss = { recordedAt: "2026-07-20T00:00:00.000Z", loosePrompt: "add a thing", missing: "state the exact anti-pattern" }; + + it("parses valid records, keeping category only when present and non-empty", () => { + const parsed = parseDraftingMisses(JSON.stringify([validMiss, { ...validMiss, category: "unstated-anti-pattern" }, { ...validMiss, category: "" }])); + expect(parsed).toHaveLength(3); + expect(parsed[0]).toEqual(validMiss); + expect(parsed[1]!.category).toBe("unstated-anti-pattern"); + expect(parsed[2]!.category).toBeUndefined(); + }); + + it("fails loud on invalid JSON, a non-array root, and malformed entries — a broken lesson file must stop the draft", () => { + expect(() => parseDraftingMisses("not json")).toThrow(/not valid JSON/); + expect(() => parseDraftingMisses('{"a":1}')).toThrow(/must be a JSON array/); + expect(() => parseDraftingMisses(JSON.stringify([null]))).toThrow(/miss #0 is malformed/); + expect(() => parseDraftingMisses(JSON.stringify([{ ...validMiss, missing: "" }]))).toThrow(/miss #0 is malformed/); + expect(() => parseDraftingMisses(JSON.stringify([validMiss, { recordedAt: "2026-07-20", loosePrompt: 5, missing: "x" }]))).toThrow(/miss #1 is malformed/); + expect(() => parseDraftingMisses(JSON.stringify([{ ...validMiss, recordedAt: "" }]))).toThrow(/miss #0 is malformed/); + }); + + it("shares one default misses-file location with the CLIs", () => { + expect(DEFAULT_DRAFTING_MISSES_FILE).toBe("scripts/drafting-misses.json"); + }); +}); + +describe("issue-drafting draftIssueBody misses checklist (#8118)", () => { + const miss = (overrides: Partial = {}): DraftingMiss => ({ + recordedAt: "2026-07-20T00:00:00.000Z", + loosePrompt: "add a thing", + missing: "verify the exact current function signature against the checkout, not memory", + ...overrides, + }); + + it("renders no checklist section when no misses are supplied (default and explicit empty)", () => { + expect(draftIssueBody("extend detectChangedThresholds", CORPUS).body).not.toContain("Pre-publish checklist"); + expect(draftIssueBody("extend detectChangedThresholds", CORPUS, { misses: [] }).body).not.toContain("Pre-publish checklist"); + }); + + it("renders every recorded miss as a checklist line the maintainer must resolve and delete", () => { + const result = draftIssueBody("extend detectChangedThresholds", CORPUS, { + misses: [miss(), miss({ category: "unstated-anti-pattern", missing: "name what does NOT satisfy the issue" })], + }); + expect(result.body).toContain("## Pre-publish checklist — learned from recorded drafting misses. Resolve each item, then DELETE this section before publishing."); + expect(result.body).toContain("- [ ] one-off: verify the exact current function signature against the checkout, not memory"); + expect(result.body).toContain("- [ ] unstated-anti-pattern: name what does NOT satisfy the issue"); + }); + + it("collapses same-category repeats into one counted line carrying the most recent lesson", () => { + const result = draftIssueBody("extend detectChangedThresholds", CORPUS, { + misses: [ + miss({ category: "unverified-signature", recordedAt: "2026-07-19T00:00:00.000Z", missing: "older lesson wording" }), + miss({ category: "unverified-signature", recordedAt: "2026-07-21T00:00:00.000Z", missing: "newer lesson wording" }), + miss({ category: "unverified-signature", recordedAt: "2026-07-20T00:00:00.000Z", missing: "middle lesson wording" }), + ], + }); + expect(result.body).toContain("- [ ] unverified-signature (recorded 3×): newer lesson wording"); + expect(result.body).not.toContain("older lesson wording"); + }); + + it("keeps uncategorized misses one line per distinct lesson, sorted deterministically", () => { + const result = draftIssueBody("extend detectChangedThresholds", CORPUS, { + misses: [miss({ missing: "zeta lesson" }), miss({ missing: "alpha lesson" }), miss({ missing: "alpha lesson" })], + }); + const checklistLines = result.body.split("\n").filter((line) => line.startsWith("- [ ] one-off")); + expect(checklistLines).toEqual(["- [ ] one-off (recorded 2×): alpha lesson", "- [ ] one-off: zeta lesson"]); + }); +});