W3-03c: crash/power-loss drill kit

docs/drills/crash-drill.md

@@ -0,0 +1,38 @@
+# Absturz-Drill (W3-03c, Matrixzeile 21)
+
+Du prüfst, ob ProjectA nach einem harten Abbruch nichts Unklares zurücklässt:
+Der unterbrochene Lauf muss auf „reconciling" (wird abgeglichen) stehen, und es
+darf kein neuer Lauf für dieselbe Aufgabe starten, bevor Prozess und Datenbank
+übereinstimmen. Deine Datenbank wird nur über eine Kopie **gelesen**. Das
+Beenden des Prozesses ist der vereinbarte Ersatz für einen Stromausfall.
+Dauer: etwa 10 Minuten je Übergang.
+
+Voraussetzung: installiertes ProjectA, PowerShell 7, Node 24, dieses Repository.
+Die Übergänge „beansprucht", „Absicht notiert", „Prüfpunkt" und „Abschluss"
+gibt es nur bei Development-Läufen (Continuous AN, bis M4 gesperrt). Ohne
+Continuous findet der Drill keinen offenen Lauf und meldet `FEHLER`: das ist
+dann kein Bestanden, sondern „nicht abgedeckt".
+
+1. **Version notieren** (ProjectA, „Info"); du gibst sie als `-AppVersion` an.
+2. **Übergang erreichen.** Starte die Arbeit und warte auf den Moment: zum
+   Beispiel „Prozess startet" oder „Eingabe wird übergeben". Du hast nur wenige
+   Sekunden.
+3. **Prozess hart beenden.** Task-Manager, `projecta` wählen, „Task beenden"
+   (bei mehreren Einträgen alle). Nicht über das Menü schließen.
+4. **Zustand sichern.** In PowerShell 7 im Repository-Ordner, ProjectA noch
+   geschlossen:
+   `pwsh scripts/drills/crash-drill.ps1 -Phase before -Transition process-starting -AppVersion 1.4.1 -OutDir C:\Belege\crash-vorher`
+   Erlaubt für `-Transition`: claimed, intent, process-starting, input-delivery,
+   checkpoint, completion. Erwartet: eine Zeile `OK`, `Ergebnis: pass`.
+5. **ProjectA neu starten** und etwa 2 Minuten warten, bis die App ruhig ist.
+   Niemanden einen neuen Lauf starten lassen.
+6. **Prüfen.**
+   `pwsh scripts/drills/crash-drill.ps1 -Phase after -BeforeDir C:\Belege\crash-vorher -AppVersion 1.4.1 -OutDir C:\Belege\crash-nachher`
+   Erwartet: zwei Zeilen `OK` und `Ergebnis: pass`. Bei `FEHLER` beide Ordner
+   behalten und melden.
+7. **Beleg:** `crash-nachher` enthält `manifest.json`, Zustand vorher/nachher
+   und Prozessliste, alles geschwärzt. Wiederhole 2–6 für jeden Übergang in
+   einem neuen, leeren Ordner.
+
+Nicht abgedeckt: echter Stromausfall, Übergänge ohne Continuous, Zeitverlauf
+der Wiederherstellung (nur ein Schnappschuss).

scripts/drills/crash-drill.mjs

@@ -0,0 +1,25 @@
+// CLI entry of the crash drill (see docs/drills/crash-drill.md): phase before|after.
+import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { parseArgs } from 'node:util';
+import { fileURLToPath } from 'node:url';
+import { defaultOutDir } from '../lib/drill-kit.mjs';
+import { runAfter, runBefore } from '../lib/crash-drill.mjs';
+
+const { positionals: [phase], values: v } = parseArgs({
+  allowPositionals: true,
+  options: {
+    'app-dir': { type: 'string' }, out: { type: 'string' }, 'app-version': { type: 'string' }, commit: { type: 'string' },
+    'process-list': { type: 'string' }, transition: { type: 'string' }, 'before-dir': { type: 'string' },
+  },
+});
+if (!['before', 'after'].includes(phase) || !v['app-dir']) { console.error('FEHLER: crash-drill.mjs before|after --app-dir <ordner>'); process.exit(2); }
+const outDir = v.out ?? defaultOutDir(`crash-drill-${phase}`);
+const commit = v.commit ?? execFileSync('git', ['rev-parse', 'HEAD'], { cwd: fileURLToPath(new URL('../..', import.meta.url)), encoding: 'utf8' }).trim();
+const common = { appDir: v['app-dir'], outDir, appVersion: v['app-version'], commit, processList: v['process-list'] ? readFileSync(v['process-list'], 'utf8') : '' };
+const manifest = phase === 'before'
+  ? runBefore({ ...common, transition: v.transition })
+  : runAfter({ ...common, beforeDir: v['before-dir'] ?? '' });
+for (const s of manifest.steps) console.log(`${s.exitCode === 0 ? 'OK    ' : 'FEHLER'} ${s.n}. ${s.name}${s.detail ? ` - ${s.detail}` : ''}`);
+console.log(`Ergebnis: ${manifest.result}. Beleg-Ordner: ${outDir}`);
+process.exit(manifest.result === 'pass' ? 0 : 1);

scripts/drills/crash-drill.ps1

@@ -0,0 +1,26 @@
+# Crash drill launcher (PowerShell 7); steps: docs/drills/crash-drill.md
+param(
+    [Parameter(Mandatory)][ValidateSet("before", "after")][string]$Phase,
+    [string]$OutDir,
+    [string]$BeforeDir,
+    [string]$Transition,
+    [string]$AppDir = (Join-Path $env:APPDATA "com.projecta.app"),
+    [string]$AppVersion = "unknown"
+)
+$ErrorActionPreference = "Stop"
+$repo = Split-Path -Parent (Split-Path -Parent $PSScriptRoot)
+if (-not (Get-Command node -ErrorAction SilentlyContinue)) { throw "Node 24+ wird gebraucht (node nicht gefunden)" }
+
+# Process list for the bundle (names only, no command lines).
+$procFile = Join-Path ([IO.Path]::GetTempPath()) ("projecta-procs-" + [guid]::NewGuid().ToString("n") + ".txt")
+Get-Process | Sort-Object ProcessName | Select-Object Id, ProcessName | Out-String | Set-Content $procFile
+
+$argList = @($Phase, "--app-dir", $AppDir, "--process-list", $procFile, "--app-version", $AppVersion)
+if ($OutDir) { $argList += @("--out", $OutDir) }
+if ($BeforeDir) { $argList += @("--before-dir", $BeforeDir) }
+if ($Transition) { $argList += @("--transition", $Transition) }
+
+& node (Join-Path $repo "scripts/drills/crash-drill.mjs") @argList
+$code = $LASTEXITCODE
+Remove-Item $procFile -ErrorAction SilentlyContinue
+exit $code

scripts/lib/crash-drill.mjs

@@ -0,0 +1,81 @@
+// W3-03c: crash drill. Snapshot run/launch/delivery state from a temp copy of the
+// live database (the live files are never opened), then judge the restart.
+import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { createBundle, sha256File } from './drill-kit.mjs';
+export const TRANSITIONS = ['claimed', 'intent', 'process-starting', 'input-delivery', 'checkpoint', 'completion'];
+const OPEN = ['intent', 'launched'];
+const QUERIES = {
+  runs: 'SELECT id, task_id, status, claim_fence, process_id, terminal_detail FROM development_runs ORDER BY id',
+  launches: 'SELECT run_id, state, exit_code FROM development_launches ORDER BY run_id',
+  deliveries: 'SELECT run_id, state FROM development_deliveries ORDER BY run_id',
+  checkpoints: 'SELECT run_id, COUNT(*) AS n FROM development_capture_checkpoints GROUP BY run_id ORDER BY run_id',
+};
+export function snapshotState(dbPath) {
+  const tmp = mkdtempSync(join(tmpdir(), 'projecta-crash-'));
+  try {
+    for (const s of ['', '-wal', '-shm']) if (existsSync(dbPath + s)) copyFileSync(dbPath + s, join(tmp, `projecta.db${s}`));
+    const con = new DatabaseSync(join(tmp, 'projecta.db'), { readOnly: true });
+    try {
+      const tables = new Set(con.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map((r) => r.name));
+      return Object.fromEntries(Object.entries(QUERIES).map(([k, sql]) => [k, tables.has(sql.match(/FROM (\w+)/)[1]) ? con.prepare(sql).all().map((r) => ({ ...r })) : []]));
+    } finally { con.close(); }
+  } finally { rmSync(tmp, { recursive: true, force: true }); }
+}
+// Judge a restart. `before` is the state left by the kill, `after` the state once
+// the restarted app has settled. Every problem is one line; none means pass.
+export function judgeRestart(before, after) {
+  const problems = [];
+  const open = before.runs.filter((r) => OPEN.includes(r.status));
+  if (!open.length) problems.push('no open run (intent/launched) at the kill: the interrupt hit no transition');
+  const now = new Map(after.runs.map((r) => [r.id, r]));
+  for (const r of open) {
+    const a = now.get(r.id);
+    if (!a) problems.push(`run ${r.id} vanished`);
+    else if (a.status === 'reconciling') continue;
+    else problems.push(`run ${r.id}: ${r.status} -> ${a.status}; expected reconciling`);
+  }
+  const blocked = new Set(after.runs.filter((r) => r.status === 'reconciling').map((r) => r.task_id));
+  const known = new Set(before.runs.map((r) => r.id));
+  for (const r of after.runs) if (!known.has(r.id) && blocked.has(r.task_id)) problems.push(`redispatch: new run ${r.id} for task ${r.task_id} while an earlier run is reconciling`);
+  const delivery = new Map(after.deliveries.map((d) => [d.run_id, d.state]));
+  for (const d of before.deliveries) if (d.state === 'started' && delivery.get(d.run_id) === 'enqueued') problems.push(`run ${d.run_id}: delivery went started -> enqueued without a live process`);
+  for (const r of before.runs) if (!OPEN.includes(r.status) && now.get(r.id)?.status !== r.status) problems.push(`run ${r.id}: ${r.status} -> ${now.get(r.id)?.status} (silent change)`);
+  return problems;
+}
+export function runBefore({ appDir, outDir, appVersion, commit, transition, processList = '' }) {
+  const bundle = createBundle({ outDir, drill: 'crash-before', appVersion, commit });
+  const live = join(resolve(appDir), 'projecta.db');
+  if (!TRANSITIONS.includes(transition)) bundle.step('check --transition', { exitCode: 1, detail: `one of: ${TRANSITIONS.join(', ')}` });
+  else if (!existsSync(live)) bundle.step('find live database', { exitCode: 1, detail: `${live} not found` });
+  else {
+    const state = snapshotState(live);
+    bundle.addFile('state.json', JSON.stringify({ transition, ...state }, null, 2));
+    bundle.step('snapshot state after the kill (copy, read-only)', { detail: `transition ${transition}; ${state.runs.length} runs` });
+  }
+  if (processList) bundle.addFile('processes-before.txt', processList);
+  return bundle.finish(NOT_COVERED);
+}
+export function runAfter({ appDir, beforeDir, outDir, appVersion, commit, processList = '' }) {
+  const bundle = createBundle({ outDir, drill: 'crash', appVersion, commit });
+  const live = join(resolve(appDir), 'projecta.db');
+  const stateFile = join(beforeDir, 'state.json');
+  if (!existsSync(stateFile) || !existsSync(live)) { bundle.step('find inputs', { exitCode: 1, detail: 'state.json of the before step or the database is missing' }); return bundle.finish(NOT_COVERED); }
+  const hash = sha256File(live);
+  const before = JSON.parse(readFileSync(stateFile, 'utf8'));
+  const after = snapshotState(live);
+  bundle.addFile('state-before.json', JSON.stringify(before, null, 2));
+  bundle.addFile('state-after.json', JSON.stringify(after, null, 2));
+  if (processList) bundle.addFile('processes-after.txt', processList);
+  const problems = judgeRestart(before, after);
+  bundle.step('judge restart: reconciling, no redispatch, no silent change', { command: `transition ${before.transition}`, exitCode: problems.length ? 1 : 0, detail: problems.join('; ') || 'ok' });
+  bundle.step('live database file untouched by the script', { exitCode: sha256File(live) === hash ? 0 : 1, detail: 'main file hash equal during the check (the running app may still write its own WAL)' });
+  return bundle.finish(NOT_COVERED);
+}
+export const NOT_COVERED = [
+  'Real power loss: killing the process is the accepted proxy; OS/disk caches are not exercised.',
+  'Transitions claimed, intent recorded, checkpoint and completion only exist for development runs, which need Continuous ON (frozen until M4); with Continuous OFF the drill only proves the failure to find an open run.',
+  'Whether the restarted app has finished its own recovery pass is judged from one snapshot, not a time series.',
+];

scripts/lib/crash-drill.test.mjs

@@ -0,0 +1,46 @@
+import assert from 'node:assert/strict';
+import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import test from 'node:test';
+import { DatabaseSync } from 'node:sqlite';
+import { judgeRestart, runAfter, runBefore, snapshotState } from './crash-drill.mjs';
+const run = (id, status, extra = {}) => ({ id, task_id: 't1', status, claim_fence: 1, process_id: null, terminal_detail: null, ...extra });
+const state = (runs, deliveries = []) => ({ runs, launches: [], deliveries, checkpoints: [] });
+test('an open run that becomes reconciling passes', () => {
+  assert.deepEqual(judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'reconciling')])), []);
+});
+test('an open run left open or completed silently fails', () => {
+  assert.equal(judgeRestart(state([run('r1', 'intent')]), state([run('r1', 'launched')])).length, 1);
+  assert.equal(judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'completed')])).length, 1);
+});
+test('a failed run cannot pass because it releases the redispatch block', () => {
+  const after = state([run('r1', 'failed', { terminal_detail: 'startup gave up' }), run('r2', 'intent')]);
+  assert.match(judgeRestart(state([run('r1', 'launched')]), after).join(), /expected reconciling/);
+});
+test('a new run for a task with a reconciling run is a redispatch', () => {
+  const p = judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'reconciling'), run('r2', 'intent')]));
+  assert.match(p.join(), /redispatch: new run r2/);
+});
+test('a delivery must not advance without a process and a drill needs an open run', () => {
+  const b = state([run('r1', 'launched')], [{ run_id: 'r1', state: 'started' }]);
+  assert.match(judgeRestart(b, state([run('r1', 'reconciling')], [{ run_id: 'r1', state: 'enqueued' }])).join(), /started -> enqueued/);
+  assert.match(judgeRestart(state([run('r1', 'completed')]), state([run('r1', 'completed')])).join(), /no open run/);
+});
+test('before and after bundles read a copy and pass for a reconciled run', () => {
+  const appDir = mkdtempSync(join(tmpdir(), 'crash-app-'));
+  const db = join(appDir, 'projecta.db');
+  const con = new DatabaseSync(db);
+  con.exec("CREATE TABLE development_runs (id TEXT, task_id TEXT, status TEXT, claim_fence INTEGER, process_id INTEGER, terminal_detail TEXT); INSERT INTO development_runs VALUES ('r1','t1','launched',1,42,NULL)");
+  assert.deepEqual(snapshotState(db).deliveries, []);
+  const dir = mkdtempSync(join(tmpdir(), 'crash-out-'));
+  const beforeDir = join(dir, 'b');
+  const m1 = runBefore({ appDir, outDir: beforeDir, transition: 'intent', processList: `token=${'q'.repeat(20)}` });
+  assert.equal(m1.result, 'pass');
+  assert.ok(!readFileSync(join(beforeDir, 'processes-before.txt'), 'utf8').includes('qqqqq'));
+  assert.equal(runAfter({ appDir, beforeDir, outDir: join(dir, 'a') }).result, 'fail');
+  con.exec("UPDATE development_runs SET status='reconciling'"); con.close();
+  const m2 = runAfter({ appDir, beforeDir, outDir: join(dir, 'a2') });
+  assert.equal(m2.result, 'pass', JSON.stringify(m2.steps));
+  assert.equal(runBefore({ appDir, outDir: join(dir, 'x'), transition: 'bogus' }).result, 'fail');
+});