Conflict-proof the generated patches, CHANGELOG and core docs (Phase 1, PR A)

.claude/skills/fix-issue/SKILL.md

@@ -37,7 +37,7 @@ node scripts/ds-bot-patches.mjs generate && node scripts/ds-bot-patches.mjs chec
 ```
 
 - An owned file ([docs/subsystems/](../../../docs/subsystems/README.md)) changed → update that subsystem doc in the same PR; `Docs-not-needed:` only when the doc truly says nothing about the change.
-- A `core/` file changed for the first time must join a topic in `patches/ds-bot/topics.json` before `generate`.
+- An upstream file (one that `ds-bot/` has) edited in `core/` for the first time must join a topic in `patches/ds-bot/topics.json` before `generate`. A new file needs no entry; `generate` stores it whole in `patches/ds-bot/files/`.
 - Compare lint warnings against base: new warnings are findings.
 - `index.js`/`src/host/` changes → run the smoke test as well.
 

.github/agent/risk-rules.json

@@ -31,7 +31,7 @@
     },
     "T1": {
       "meaning": "low risk: client code, tests, generated files",
-      "paths": ["core/src/client/**", "core/client.js", "patches/**", "core/test/**", "plugins/*/test/**", "CHANGELOG.md"]
+      "paths": ["core/src/client/**", "core/client.js", "patches/**", "core/test/**", "plugins/*/test/**", "CHANGELOG.md", "changelog.d/**"]
     },
     "T0": {
       "meaning": "lowest risk: documentation",

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 Fixes #
 
 ## What changed
-<!-- Observable behavior, before and after. Keep this short: the PR title and body become the squash commit message. -->
+<!-- Observable behavior, before and after. Keep this short: the PR title and body become the squash commit message. A user-visible change also adds changelog.d/<name>.<kind>.md (not CHANGELOG.md). -->
 
 ## Verification at the tip
 Tip SHA:

.github/scripts/changelog.mjs

@@ -0,0 +1,124 @@
+// CHANGELOG entries live in one fragment file per change (changelog.d/<name>.<kind>.md), so two PRs
+// never edit the same lines of CHANGELOG.md. At release time `release` folds them into a section.
+//
+//   node .github/scripts/changelog.mjs check                       fragments are well formed; CHANGELOG.md has [Unreleased]
+//   node .github/scripts/changelog.mjs preview                     print what the fragments would add (to stdout)
+//   node .github/scripts/changelog.mjs release <version> [--date YYYY-MM-DD] [--dry-run]
+//                                                                  move [Unreleased] and the fragments into "## [version] - date",
+//                                                                  delete the fragment files, leave [Unreleased] empty
+//
+// No network, no shell, no environment variables: it reads and writes CHANGELOG.md and changelog.d/ only.
+import { existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+export const KINDS = ['added', 'changed', 'deprecated', 'removed', 'fixed', 'security']
+const HEADING = kind => `### ${kind[0].toUpperCase()}${kind.slice(1)}`
+const NAME = new RegExp(`^[a-z0-9][a-z0-9_-]*(?:\\.[a-z0-9_-]+)*\\.(${KINDS.join('|')})\\.md$`)
+
+// [{ file, kind, text }] sorted by file name; throws one Error listing every problem.
+export function readFragments(dir) {
+  if (!existsSync(dir)) return []
+  const problems = []
+  const out = []
+  for (const file of readdirSync(dir).sort()) {
+    if (file === 'README.md' || file === '.gitkeep') continue
+    const match = NAME.exec(file)
+    if (!match) { problems.push(`${file}: name must be <slug>.<kind>.md with kind one of ${KINDS.join(', ')} (lower case)`); continue }
+    const text = readFileSync(join(dir, file), 'utf8').replace(/\r\n/g, '\n').trim()
+    if (!text) problems.push(`${file}: empty`)
+    else if (/^(- |\* |#)/.test(text)) problems.push(`${file}: write the entry text only (no leading "- " or heading); the bullet is added when it is assembled`)
+    else if (/\n\s*\n/.test(text)) problems.push(`${file}: one entry per file, no blank lines inside it`)
+    else out.push({ file, kind: match[1], text })
+  }
+  if (problems.length) throw new Error(problems.join('\n'))
+  return out
+}
+
+const bullet = text => `- ${text.split('\n').map((l, i) => (i ? `  ${l.trim()}` : l.trim())).join('\n')}`
+
+// "### Kind" blocks -> Map(kind -> bullet text lines); text before the first heading is kept as `lead`.
+export function parseSection(body) {
+  const kinds = new Map()
+  let lead = ''
+  let current = null
+  for (const line of body.split('\n')) {
+    const heading = /^### (.+?)\s*$/.exec(line)
+    if (heading) { current = heading[1]; if (!kinds.has(current)) kinds.set(current, []); continue }
+    if (current === null) lead += `${line}\n`
+    else kinds.get(current).push(line)
+  }
+  for (const [name, lines] of kinds) kinds.set(name, lines.join('\n').trim().split('\n').filter((l, i, all) => all.length > 1 || l))
+  return { lead: lead.trim(), kinds }
+}
+
+export function renderKinds(kinds) {
+  const known = KINDS.map(k => HEADING(k).slice(4))
+  const order = [...known.filter(k => kinds.has(k)), ...[...kinds.keys()].filter(k => !known.includes(k))]
+  return order.map(k => `### ${k}\n\n${kinds.get(k).join('\n').trim()}`).join('\n\n')
+}
+
+// Fragments as "### Kind" blocks (Keep a Changelog order).
+export function assemble(fragments) {
+  const kinds = new Map()
+  for (const f of fragments) {
+    const name = HEADING(f.kind).slice(4)
+    if (!kinds.has(name)) kinds.set(name, [])
+    kinds.get(name).push(bullet(f.text))
+  }
+  return renderKinds(kinds)
+}
+
+const UNRELEASED = /^## \[Unreleased\][ \t]*$/m
+
+// New CHANGELOG.md text: [Unreleased] emptied, a "## [version] - date" section below it that holds the old
+// [Unreleased] entries first and then the fragments, merged per kind.
+export function release(changelog, fragments, version, date) {
+  const start = changelog.search(UNRELEASED)
+  if (start < 0) throw new Error('CHANGELOG.md has no "## [Unreleased]" section')
+  if (!/^\d+\.\d+\.\d+([-+][0-9A-Za-z.-]+)?$/.test(version)) throw new Error(`"${version}" is not a version like 1.2.3`)
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) throw new Error(`"${date}" is not a date like 2026-10-15`)
+  if (new RegExp(`^## \\[${version.replace(/[.+]/g, '\\$&')}\\]`, 'm').test(changelog)) throw new Error(`CHANGELOG.md already has a section for ${version}`)
+  const afterHeading = changelog.indexOf('\n', start) + 1
+  const next = changelog.slice(afterHeading).search(/^## \[/m)
+  const end = next < 0 ? changelog.length : afterHeading + next
+  const { lead, kinds } = parseSection(changelog.slice(afterHeading, end))
+  for (const f of fragments) {
+    const name = HEADING(f.kind).slice(4)
+    if (!kinds.has(name)) kinds.set(name, [])
+    kinds.get(name).push(bullet(f.text))
+  }
+  if (!kinds.size && !lead) throw new Error('nothing to release: [Unreleased] is empty and changelog.d/ has no fragments')
+  const section = `## [${version}] - ${date}\n\n${lead ? `${lead}\n\n` : ''}${renderKinds(kinds)}\n\n`
+  return `${changelog.slice(0, afterHeading)}\n${section}${changelog.slice(end)}`
+}
+
+function main(argv) {
+  const root = process.env.CHANGELOG_ROOT ?? join(fileURLToPath(import.meta.url), '..', '..', '..')
+  const dir = join(root, 'changelog.d')
+  const file = join(root, 'CHANGELOG.md')
+  const [command, ...rest] = argv
+  const fail = message => { console.error(`changelog: ${message}`); process.exit(1) }
+  try {
+    if (command === 'check') {
+      const fragments = readFragments(dir)
+      if (!UNRELEASED.test(readFileSync(file, 'utf8'))) fail('CHANGELOG.md has no "## [Unreleased]" section')
+      console.log(`changelog: ${fragments.length} fragment(s) in changelog.d/ are well formed`)
+    } else if (command === 'preview') {
+      console.log(assemble(readFragments(dir)))
+    } else if (command === 'release') {
+      const version = rest.find(a => !a.startsWith('--') && rest[rest.indexOf(a) - 1] !== '--date')
+      const dateAt = rest.indexOf('--date')
+      const date = dateAt >= 0 ? rest[dateAt + 1] : new Date().toISOString().slice(0, 10)
+      if (!version) fail('usage: changelog.mjs release <version> [--date YYYY-MM-DD] [--dry-run]')
+      const fragments = readFragments(dir)
+      const text = release(readFileSync(file, 'utf8'), fragments, version, date)
+      if (rest.includes('--dry-run')) { console.log(text.split('\n').slice(0, 60).join('\n')); return }
+      writeFileSync(file, text)
+      for (const f of fragments) rmSync(join(dir, f.file))
+      console.log(`changelog: [${version}] - ${date} written, ${fragments.length} fragment(s) removed. Review, then commit CHANGELOG.md and changelog.d/.`)
+    } else fail('usage: changelog.mjs check | preview | release <version> [--date YYYY-MM-DD] [--dry-run]')
+  } catch (error) { fail(error.message) }
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) main(process.argv.slice(2))

.github/scripts/changelog.test.mjs

@@ -0,0 +1,95 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, existsSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { assemble, release, readFragments } from './changelog.mjs'
+
+const CHANGELOG = `# Changelog
+
+Intro.
+
+## [Unreleased]
+
+### Changed
+
+- Old hand-written entry.
+
+## [0.1.0] - 2026-10-14
+
+### Added
+
+- First.
+`
+
+const tmp = () => mkdtempSync(join(tmpdir(), 'changelog-'))
+const fragments = dir => { mkdirSync(dir, { recursive: true }); return (name, text) => writeFileSync(join(dir, name), text) }
+
+test('readFragments: sorted, kind from the name, README ignored', () => {
+  const dir = tmp()
+  const put = fragments(dir)
+  put('b-second.fixed.md', 'Second.\n')
+  put('a-first.added.md', 'First.\n')
+  put('README.md', 'ignored')
+  assert.deepEqual(readFragments(dir).map(f => [f.file, f.kind, f.text]), [['a-first.added.md', 'added', 'First.'], ['b-second.fixed.md', 'fixed', 'Second.']])
+  assert.deepEqual(readFragments(join(dir, 'missing')), [])
+  rmSync(dir, { recursive: true })
+})
+
+test('readFragments: every malformed file is reported', () => {
+  const dir = tmp()
+  const put = fragments(dir)
+  put('no-kind.md', 'x')
+  put('Upper.added.md', 'x')
+  put('empty.added.md', '\n')
+  put('bullet.added.md', '- already a bullet')
+  put('two.added.md', 'one\n\ntwo')
+  put('unknown.misc.md', 'x')
+  assert.throws(() => readFragments(dir), (error) => ['no-kind.md', 'Upper.added.md', 'empty.added.md', 'bullet.added.md', 'two.added.md', 'unknown.misc.md'].every(n => error.message.includes(n)))
+  rmSync(dir, { recursive: true })
+})
+
+test('assemble: Keep a Changelog order, one bullet per fragment, continuation lines indented', () => {
+  const text = assemble([
+    { file: 'a.fixed.md', kind: 'fixed', text: 'Fix A.' },
+    { file: 'b.added.md', kind: 'added', text: 'Add B\nsecond line.' },
+    { file: 'c.fixed.md', kind: 'fixed', text: 'Fix C.' },
+  ])
+  assert.equal(text, '### Added\n\n- Add B\n  second line.\n\n### Fixed\n\n- Fix A.\n- Fix C.')
+})
+
+test('release: Unreleased is emptied; old entries come first, fragments merge per kind', () => {
+  const out = release(CHANGELOG, [{ kind: 'changed', text: 'New change.' }, { kind: 'fixed', text: 'New fix.' }], '0.1.1', '2026-10-20')
+  assert.match(out, /## \[Unreleased\]\n\n## \[0\.1\.1\] - 2026-10-20\n\n### Changed\n\n- Old hand-written entry\.\n- New change\.\n\n### Fixed\n\n- New fix\.\n\n## \[0\.1\.0\]/)
+  assert.ok(out.startsWith('# Changelog\n\nIntro.\n\n## [Unreleased]\n'))
+  assert.ok(out.endsWith('- First.\n'))
+})
+
+test('release: refuses bad input and a duplicate version', () => {
+  assert.throws(() => release('# x\n', [], '0.1.1', '2026-10-20'), /Unreleased/)
+  assert.throws(() => release(CHANGELOG, [], 'v0.1.1', '2026-10-20'), /not a version/)
+  assert.throws(() => release(CHANGELOG, [], '0.1.1', 'today'), /not a date/)
+  assert.throws(() => release(CHANGELOG, [], '0.1.0', '2026-10-20'), /already has/)
+  assert.throws(() => release('## [Unreleased]\n\n## [0.1.0] - x\n', [], '0.1.1', '2026-10-20'), /nothing to release/)
+})
+
+test('CLI: check, preview, then release writes the file and removes the fragments', () => {
+  const root = tmp()
+  writeFileSync(join(root, 'CHANGELOG.md'), CHANGELOG)
+  fragments(join(root, 'changelog.d'))('7-thing.added.md', 'Thing (#7).\n')
+  const run = (...args) => execFileSync(process.execPath, [new URL('./changelog.mjs', import.meta.url).pathname, ...args], { env: { ...process.env, CHANGELOG_ROOT: root }, encoding: 'utf8' })
+  assert.match(run('check'), /1 fragment/)
+  assert.match(run('preview'), /### Added\n\n- Thing \(#7\)\./)
+  run('release', '0.2.0', '--date', '2026-11-01', '--dry-run')
+  assert.ok(existsSync(join(root, 'changelog.d', '7-thing.added.md')))
+  run('release', '0.2.0', '--date', '2026-11-01')
+  assert.ok(!existsSync(join(root, 'changelog.d', '7-thing.added.md')))
+  assert.match(readFileSync(join(root, 'CHANGELOG.md'), 'utf8'), /## \[0\.2\.0\] - 2026-11-01\n\n### Added\n\n- Thing \(#7\)\.\n\n### Changed\n\n- Old hand-written entry\./)
+  rmSync(root, { recursive: true })
+})
+
+test('the repo: changelog.d/ is valid and CHANGELOG.md keeps its Unreleased heading', () => {
+  readFragments(new URL('../../changelog.d', import.meta.url).pathname)
+  assert.match(readFileSync(new URL('../../CHANGELOG.md', import.meta.url), 'utf8'), /^## \[Unreleased\]$/m)
+})

.github/scripts/ds-bot-patches.test.mjs

@@ -0,0 +1,73 @@
+// The generated patch format (scripts/ds-bot-patches.mjs): header lines carry no per-edit data.
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { execFileSync } from 'node:child_process'
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { normalizeDiff } from '../../scripts/ds-bot-patches.mjs'
+
+const lines = (n, edits = {}) => Array.from({ length: n }, (_, i) => edits[i + 1] ?? `line ${i + 1}`).join('\n') + '\n'
+
+// What generate does for one file: `git diff --no-index` of ds-bot/x against core/x, then normalize.
+function diffOf(before, after, opts) {
+  const dir = mkdtempSync(join(tmpdir(), 'patch-fmt-'))
+  try {
+    mkdirSync(join(dir, 'ds-bot'))
+    mkdirSync(join(dir, 'core'))
+    writeFileSync(join(dir, 'ds-bot', 'x.js'), before)
+    writeFileSync(join(dir, 'core', 'x.js'), after)
+    let raw
+    try { raw = execFileSync('git', ['diff', '--no-index', '--no-color', '--', 'ds-bot/x.js', 'core/x.js'], { cwd: dir, encoding: 'utf8' }) } catch (error) { raw = error.stdout }
+    return normalizeDiff(raw, opts)
+  } finally { rmSync(dir, { recursive: true, force: true }) }
+}
+
+function applies(before, patch) {
+  const dir = mkdtempSync(join(tmpdir(), 'patch-fmt-'))
+  try {
+    writeFileSync(join(dir, 'x.js'), before)
+    writeFileSync(join(dir, 'p.patch'), patch)
+    execFileSync('git', ['apply', '--whitespace=nowarn', '-p1', 'p.patch'], { cwd: dir, stdio: 'pipe' })
+    return readFileSync(join(dir, 'x.js'), 'utf8')
+  } finally { rmSync(dir, { recursive: true, force: true }) }
+}
+
+const base = lines(200)
+const top = { 3: 'line 3\nadded at the top\nand a second line' }
+const bottom = { 190: 'line 190\nadded at the bottom' }
+
+test('no index line, no ds-bot/ or core/ prefix, no per-file counts of other edits', () => {
+  const patch = diffOf(base, lines(200, top))
+  assert.doesNotMatch(patch, /^index /m)
+  assert.match(patch, /^diff --git a\/x\.js b\/x\.js$/m)
+  assert.match(patch, /^--- a\/x\.js$/m)
+  assert.match(patch, /^\+\+\+ b\/x\.js$/m)
+})
+
+test('a hunk header does not depend on edits in other hunks', () => {
+  const both = { ...top, ...bottom }
+  const hunks = text => text.split('\n').filter(l => l.startsWith('@@'))
+  const alone = hunks(diffOf(base, lines(200, bottom)))
+  const withTop = hunks(diffOf(base, lines(200, both)))
+  assert.equal(alone.length, 1)
+  assert.ok(withTop.includes(alone[0]), `${alone[0]} not in ${withTop}`)
+  // the exact form shifts the new-side start, which is what made two branches conflict
+  const exact = diffOf(base, lines(200, both), { exact: true }).split('\n').filter(l => l.startsWith('@@'))
+  assert.ok(!exact.includes(alone[0]))
+})
+
+test('the stored form still applies, and gives the new file', () => {
+  const both = { ...top, ...bottom }
+  assert.equal(applies(base, diffOf(base, lines(200, both))), lines(200, both))
+  assert.equal(applies(base, diffOf(base, lines(200, both), { exact: true })), lines(200, both))
+})
+
+test('hunk bodies are never rewritten, even when they look like headers', () => {
+  const tricky = ['--- a/ds-bot/x.js', '+++ b/core/x.js', 'index abc..def 100644', '@@ -1,2 +9,9 @@', 'ctx']
+  const before = tricky.join('\n') + '\n'
+  const after = ['ctx', ...tricky.slice(0, 4)].join('\n') + '\n'
+  const patch = diffOf(before, after)
+  for (const l of tricky.slice(0, 4)) assert.ok(patch.includes(l), `lost: ${l}`)
+  assert.equal(applies(before, patch), after)
+})

.github/workflows/test.yml

@@ -102,8 +102,8 @@ jobs:
         run: npx --yes -p typescript@5.9.3 tsc --noEmit --strict --skipLibCheck types/*.d.ts
       - name: Tests of the repo scripts (risk tier, labels)
         run: node --test ".github/scripts/*.test.mjs"
-      - name: Changelog has an Unreleased section
-        run: grep -q '^## \[Unreleased\]' CHANGELOG.md
+      - name: Changelog has an Unreleased section and well-formed fragments (changelog.d/)
+        run: node .github/scripts/changelog.mjs check
       - name: Lint the workflows (actionlint)
         env:
           VERSION: 1.7.7