diff --git a/docs/drills/crash-drill.md b/docs/drills/crash-drill.md new file mode 100644 index 00000000..18e37cf6 --- /dev/null +++ b/docs/drills/crash-drill.md @@ -0,0 +1,38 @@ +# Absturz-Drill (W3-03c, Matrixzeile 21) + +Du prüfst, ob ProjectA nach einem harten Abbruch nichts Unklares zurücklässt: +Der unterbrochene Lauf muss auf „reconciling" (wird abgeglichen) stehen, und es +darf kein neuer Lauf für dieselbe Aufgabe starten, bevor Prozess und Datenbank +übereinstimmen. Deine Datenbank wird nur über eine Kopie **gelesen**. Das +Beenden des Prozesses ist der vereinbarte Ersatz für einen Stromausfall. +Dauer: etwa 10 Minuten je Übergang. + +Voraussetzung: installiertes ProjectA, PowerShell 7, Node 24, dieses Repository. +Die Übergänge „beansprucht", „Absicht notiert", „Prüfpunkt" und „Abschluss" +gibt es nur bei Development-Läufen (Continuous AN, bis M4 gesperrt). Ohne +Continuous findet der Drill keinen offenen Lauf und meldet `FEHLER`: das ist +dann kein Bestanden, sondern „nicht abgedeckt". + +1. **Version notieren** (ProjectA, „Info"); du gibst sie als `-AppVersion` an. +2. **Übergang erreichen.** Starte die Arbeit und warte auf den Moment: zum + Beispiel „Prozess startet" oder „Eingabe wird übergeben". Du hast nur wenige + Sekunden. +3. **Prozess hart beenden.** Task-Manager, `projecta` wählen, „Task beenden" + (bei mehreren Einträgen alle). Nicht über das Menü schließen. +4. **Zustand sichern.** In PowerShell 7 im Repository-Ordner, ProjectA noch + geschlossen: + `pwsh scripts/drills/crash-drill.ps1 -Phase before -Transition process-starting -AppVersion 1.4.1 -OutDir C:\Belege\crash-vorher` + Erlaubt für `-Transition`: claimed, intent, process-starting, input-delivery, + checkpoint, completion. Erwartet: eine Zeile `OK`, `Ergebnis: pass`. +5. **ProjectA neu starten** und etwa 2 Minuten warten, bis die App ruhig ist. + Niemanden einen neuen Lauf starten lassen. +6. **Prüfen.** + `pwsh scripts/drills/crash-drill.ps1 -Phase after -BeforeDir C:\Belege\crash-vorher -AppVersion 1.4.1 -OutDir C:\Belege\crash-nachher` + Erwartet: zwei Zeilen `OK` und `Ergebnis: pass`. Bei `FEHLER` beide Ordner + behalten und melden. +7. **Beleg:** `crash-nachher` enthält `manifest.json`, Zustand vorher/nachher + und Prozessliste, alles geschwärzt. Wiederhole 2–6 für jeden Übergang in + einem neuen, leeren Ordner. + +Nicht abgedeckt: echter Stromausfall, Übergänge ohne Continuous, Zeitverlauf +der Wiederherstellung (nur ein Schnappschuss). diff --git a/scripts/drills/crash-drill.mjs b/scripts/drills/crash-drill.mjs new file mode 100644 index 00000000..58e03649 --- /dev/null +++ b/scripts/drills/crash-drill.mjs @@ -0,0 +1,25 @@ +// CLI entry of the crash drill (see docs/drills/crash-drill.md): phase before|after. +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { parseArgs } from 'node:util'; +import { fileURLToPath } from 'node:url'; +import { defaultOutDir } from '../lib/drill-kit.mjs'; +import { runAfter, runBefore } from '../lib/crash-drill.mjs'; + +const { positionals: [phase], values: v } = parseArgs({ + allowPositionals: true, + options: { + 'app-dir': { type: 'string' }, out: { type: 'string' }, 'app-version': { type: 'string' }, commit: { type: 'string' }, + 'process-list': { type: 'string' }, transition: { type: 'string' }, 'before-dir': { type: 'string' }, + }, +}); +if (!['before', 'after'].includes(phase) || !v['app-dir']) { console.error('FEHLER: crash-drill.mjs before|after --app-dir '); process.exit(2); } +const outDir = v.out ?? defaultOutDir(`crash-drill-${phase}`); +const commit = v.commit ?? execFileSync('git', ['rev-parse', 'HEAD'], { cwd: fileURLToPath(new URL('../..', import.meta.url)), encoding: 'utf8' }).trim(); +const common = { appDir: v['app-dir'], outDir, appVersion: v['app-version'], commit, processList: v['process-list'] ? readFileSync(v['process-list'], 'utf8') : '' }; +const manifest = phase === 'before' + ? runBefore({ ...common, transition: v.transition }) + : runAfter({ ...common, beforeDir: v['before-dir'] ?? '' }); +for (const s of manifest.steps) console.log(`${s.exitCode === 0 ? 'OK ' : 'FEHLER'} ${s.n}. ${s.name}${s.detail ? ` - ${s.detail}` : ''}`); +console.log(`Ergebnis: ${manifest.result}. Beleg-Ordner: ${outDir}`); +process.exit(manifest.result === 'pass' ? 0 : 1); diff --git a/scripts/drills/crash-drill.ps1 b/scripts/drills/crash-drill.ps1 new file mode 100644 index 00000000..67787105 --- /dev/null +++ b/scripts/drills/crash-drill.ps1 @@ -0,0 +1,26 @@ +# Crash drill launcher (PowerShell 7); steps: docs/drills/crash-drill.md +param( + [Parameter(Mandatory)][ValidateSet("before", "after")][string]$Phase, + [string]$OutDir, + [string]$BeforeDir, + [string]$Transition, + [string]$AppDir = (Join-Path $env:APPDATA "com.projecta.app"), + [string]$AppVersion = "unknown" +) +$ErrorActionPreference = "Stop" +$repo = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) +if (-not (Get-Command node -ErrorAction SilentlyContinue)) { throw "Node 24+ wird gebraucht (node nicht gefunden)" } + +# Process list for the bundle (names only, no command lines). +$procFile = Join-Path ([IO.Path]::GetTempPath()) ("projecta-procs-" + [guid]::NewGuid().ToString("n") + ".txt") +Get-Process | Sort-Object ProcessName | Select-Object Id, ProcessName | Out-String | Set-Content $procFile + +$argList = @($Phase, "--app-dir", $AppDir, "--process-list", $procFile, "--app-version", $AppVersion) +if ($OutDir) { $argList += @("--out", $OutDir) } +if ($BeforeDir) { $argList += @("--before-dir", $BeforeDir) } +if ($Transition) { $argList += @("--transition", $Transition) } + +& node (Join-Path $repo "scripts/drills/crash-drill.mjs") @argList +$code = $LASTEXITCODE +Remove-Item $procFile -ErrorAction SilentlyContinue +exit $code diff --git a/scripts/lib/crash-drill.mjs b/scripts/lib/crash-drill.mjs new file mode 100644 index 00000000..9977f54f --- /dev/null +++ b/scripts/lib/crash-drill.mjs @@ -0,0 +1,81 @@ +// W3-03c: crash drill. Snapshot run/launch/delivery state from a temp copy of the +// live database (the live files are never opened), then judge the restart. +import { copyFileSync, existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { createBundle, sha256File } from './drill-kit.mjs'; +export const TRANSITIONS = ['claimed', 'intent', 'process-starting', 'input-delivery', 'checkpoint', 'completion']; +const OPEN = ['intent', 'launched']; +const QUERIES = { + runs: 'SELECT id, task_id, status, claim_fence, process_id, terminal_detail FROM development_runs ORDER BY id', + launches: 'SELECT run_id, state, exit_code FROM development_launches ORDER BY run_id', + deliveries: 'SELECT run_id, state FROM development_deliveries ORDER BY run_id', + checkpoints: 'SELECT run_id, COUNT(*) AS n FROM development_capture_checkpoints GROUP BY run_id ORDER BY run_id', +}; +export function snapshotState(dbPath) { + const tmp = mkdtempSync(join(tmpdir(), 'projecta-crash-')); + try { + for (const s of ['', '-wal', '-shm']) if (existsSync(dbPath + s)) copyFileSync(dbPath + s, join(tmp, `projecta.db${s}`)); + const con = new DatabaseSync(join(tmp, 'projecta.db'), { readOnly: true }); + try { + const tables = new Set(con.prepare("SELECT name FROM sqlite_master WHERE type='table'").all().map((r) => r.name)); + return Object.fromEntries(Object.entries(QUERIES).map(([k, sql]) => [k, tables.has(sql.match(/FROM (\w+)/)[1]) ? con.prepare(sql).all().map((r) => ({ ...r })) : []])); + } finally { con.close(); } + } finally { rmSync(tmp, { recursive: true, force: true }); } +} +// Judge a restart. `before` is the state left by the kill, `after` the state once +// the restarted app has settled. Every problem is one line; none means pass. +export function judgeRestart(before, after) { + const problems = []; + const open = before.runs.filter((r) => OPEN.includes(r.status)); + if (!open.length) problems.push('no open run (intent/launched) at the kill: the interrupt hit no transition'); + const now = new Map(after.runs.map((r) => [r.id, r])); + for (const r of open) { + const a = now.get(r.id); + if (!a) problems.push(`run ${r.id} vanished`); + else if (a.status === 'reconciling') continue; + else problems.push(`run ${r.id}: ${r.status} -> ${a.status}; expected reconciling`); + } + const blocked = new Set(after.runs.filter((r) => r.status === 'reconciling').map((r) => r.task_id)); + const known = new Set(before.runs.map((r) => r.id)); + for (const r of after.runs) if (!known.has(r.id) && blocked.has(r.task_id)) problems.push(`redispatch: new run ${r.id} for task ${r.task_id} while an earlier run is reconciling`); + const delivery = new Map(after.deliveries.map((d) => [d.run_id, d.state])); + for (const d of before.deliveries) if (d.state === 'started' && delivery.get(d.run_id) === 'enqueued') problems.push(`run ${d.run_id}: delivery went started -> enqueued without a live process`); + for (const r of before.runs) if (!OPEN.includes(r.status) && now.get(r.id)?.status !== r.status) problems.push(`run ${r.id}: ${r.status} -> ${now.get(r.id)?.status} (silent change)`); + return problems; +} +export function runBefore({ appDir, outDir, appVersion, commit, transition, processList = '' }) { + const bundle = createBundle({ outDir, drill: 'crash-before', appVersion, commit }); + const live = join(resolve(appDir), 'projecta.db'); + if (!TRANSITIONS.includes(transition)) bundle.step('check --transition', { exitCode: 1, detail: `one of: ${TRANSITIONS.join(', ')}` }); + else if (!existsSync(live)) bundle.step('find live database', { exitCode: 1, detail: `${live} not found` }); + else { + const state = snapshotState(live); + bundle.addFile('state.json', JSON.stringify({ transition, ...state }, null, 2)); + bundle.step('snapshot state after the kill (copy, read-only)', { detail: `transition ${transition}; ${state.runs.length} runs` }); + } + if (processList) bundle.addFile('processes-before.txt', processList); + return bundle.finish(NOT_COVERED); +} +export function runAfter({ appDir, beforeDir, outDir, appVersion, commit, processList = '' }) { + const bundle = createBundle({ outDir, drill: 'crash', appVersion, commit }); + const live = join(resolve(appDir), 'projecta.db'); + const stateFile = join(beforeDir, 'state.json'); + if (!existsSync(stateFile) || !existsSync(live)) { bundle.step('find inputs', { exitCode: 1, detail: 'state.json of the before step or the database is missing' }); return bundle.finish(NOT_COVERED); } + const hash = sha256File(live); + const before = JSON.parse(readFileSync(stateFile, 'utf8')); + const after = snapshotState(live); + bundle.addFile('state-before.json', JSON.stringify(before, null, 2)); + bundle.addFile('state-after.json', JSON.stringify(after, null, 2)); + if (processList) bundle.addFile('processes-after.txt', processList); + const problems = judgeRestart(before, after); + bundle.step('judge restart: reconciling, no redispatch, no silent change', { command: `transition ${before.transition}`, exitCode: problems.length ? 1 : 0, detail: problems.join('; ') || 'ok' }); + bundle.step('live database file untouched by the script', { exitCode: sha256File(live) === hash ? 0 : 1, detail: 'main file hash equal during the check (the running app may still write its own WAL)' }); + return bundle.finish(NOT_COVERED); +} +export const NOT_COVERED = [ + 'Real power loss: killing the process is the accepted proxy; OS/disk caches are not exercised.', + 'Transitions claimed, intent recorded, checkpoint and completion only exist for development runs, which need Continuous ON (frozen until M4); with Continuous OFF the drill only proves the failure to find an open run.', + 'Whether the restarted app has finished its own recovery pass is judged from one snapshot, not a time series.', +]; diff --git a/scripts/lib/crash-drill.test.mjs b/scripts/lib/crash-drill.test.mjs new file mode 100644 index 00000000..a90caab4 --- /dev/null +++ b/scripts/lib/crash-drill.test.mjs @@ -0,0 +1,46 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; +import { judgeRestart, runAfter, runBefore, snapshotState } from './crash-drill.mjs'; +const run = (id, status, extra = {}) => ({ id, task_id: 't1', status, claim_fence: 1, process_id: null, terminal_detail: null, ...extra }); +const state = (runs, deliveries = []) => ({ runs, launches: [], deliveries, checkpoints: [] }); +test('an open run that becomes reconciling passes', () => { + assert.deepEqual(judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'reconciling')])), []); +}); +test('an open run left open or completed silently fails', () => { + assert.equal(judgeRestart(state([run('r1', 'intent')]), state([run('r1', 'launched')])).length, 1); + assert.equal(judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'completed')])).length, 1); +}); +test('a failed run cannot pass because it releases the redispatch block', () => { + const after = state([run('r1', 'failed', { terminal_detail: 'startup gave up' }), run('r2', 'intent')]); + assert.match(judgeRestart(state([run('r1', 'launched')]), after).join(), /expected reconciling/); +}); +test('a new run for a task with a reconciling run is a redispatch', () => { + const p = judgeRestart(state([run('r1', 'launched')]), state([run('r1', 'reconciling'), run('r2', 'intent')])); + assert.match(p.join(), /redispatch: new run r2/); +}); +test('a delivery must not advance without a process and a drill needs an open run', () => { + const b = state([run('r1', 'launched')], [{ run_id: 'r1', state: 'started' }]); + assert.match(judgeRestart(b, state([run('r1', 'reconciling')], [{ run_id: 'r1', state: 'enqueued' }])).join(), /started -> enqueued/); + assert.match(judgeRestart(state([run('r1', 'completed')]), state([run('r1', 'completed')])).join(), /no open run/); +}); +test('before and after bundles read a copy and pass for a reconciled run', () => { + const appDir = mkdtempSync(join(tmpdir(), 'crash-app-')); + const db = join(appDir, 'projecta.db'); + const con = new DatabaseSync(db); + con.exec("CREATE TABLE development_runs (id TEXT, task_id TEXT, status TEXT, claim_fence INTEGER, process_id INTEGER, terminal_detail TEXT); INSERT INTO development_runs VALUES ('r1','t1','launched',1,42,NULL)"); + assert.deepEqual(snapshotState(db).deliveries, []); + const dir = mkdtempSync(join(tmpdir(), 'crash-out-')); + const beforeDir = join(dir, 'b'); + const m1 = runBefore({ appDir, outDir: beforeDir, transition: 'intent', processList: `token=${'q'.repeat(20)}` }); + assert.equal(m1.result, 'pass'); + assert.ok(!readFileSync(join(beforeDir, 'processes-before.txt'), 'utf8').includes('qqqqq')); + assert.equal(runAfter({ appDir, beforeDir, outDir: join(dir, 'a') }).result, 'fail'); + con.exec("UPDATE development_runs SET status='reconciling'"); con.close(); + const m2 = runAfter({ appDir, beforeDir, outDir: join(dir, 'a2') }); + assert.equal(m2.result, 'pass', JSON.stringify(m2.steps)); + assert.equal(runBefore({ appDir, outDir: join(dir, 'x'), transition: 'bogus' }).result, 'fail'); +});