From a7626cbdc6ccf82278bda97a808cbd60776ba466 Mon Sep 17 00:00:00 2001 From: dripsmvcp <138900956+dripsmvcp@users.noreply.github.com> Date: Thu, 11 Jun 2026 01:07:26 +0900 Subject: [PATCH] feat(migrate): versioned semver schema migrations for the on-disk KB (#200) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Give the .vouch/ contract a versioned, reversible, audit-logged upgrade path so evolving the pydantic models never silently breaks a KB in the wild. vouch migrate status current schema version, target, pending migrations vouch migrate plan dry-run: every file each pending migration changes vouch migrate apply apply pending migrations (audit-logged, atomic) vouch migrate rollback reverse the most recently applied migration vouch migrate verify parse-load every artifact under the current version Migrations are data, not code: yaml manifests in a repo-root migrations/ dir, one consecutive version step each (rename / default / drop / split / merge verbs). A new .vouch/schema_version stamp (absent => baseline 0.1.0) gates load; init writes it. Safety: - atomic per file (temp + fsync + os.replace) — never a torn write - reversible: the prior content of every touched file is journalled to .vouch/migrations/rollback-.jsonl before any rewrite, so apply -> rollback is byte-equivalent - crash-safe: journal written first, schema_version bumped last, so an interrupted apply leaves the KB at its prior version with a journal to recover - refuses to run with pending proposals (won't rewrite a reviewer's queue) - state.db is disposable; the CLI rebuilds it after apply migrations.py becomes a migrations/ package: the legacy integer "format" migration (config.yaml version / KB_FORMAT_VERSION) is preserved verbatim under `vouch migrate` with no subcommand, and re-exported so existing imports and tests are unaffected. The two version axes (integer directory-format vs semver model-schema) are intentionally distinct. Closes #200 --- docs/migrations.md | 75 ++++ migrations/README.md | 57 +++ src/vouch/cli.py | 142 +++++++- src/vouch/migrations/__init__.py | 78 ++++ .../{migrations.py => migrations/_legacy.py} | 12 +- src/vouch/migrations/journal.py | 71 ++++ src/vouch/migrations/manifest.py | 152 ++++++++ src/vouch/migrations/rewriter.py | 135 +++++++ src/vouch/migrations/runner.py | 336 ++++++++++++++++++ src/vouch/migrations/schema.py | 37 ++ src/vouch/migrations/semver.py | 37 ++ src/vouch/storage.py | 9 + .../fixtures/migrations/0099-test-rename.yaml | 8 + tests/test_schema_migrations.py | 250 +++++++++++++ 14 files changed, 1389 insertions(+), 10 deletions(-) create mode 100644 docs/migrations.md create mode 100644 migrations/README.md create mode 100644 src/vouch/migrations/__init__.py rename src/vouch/{migrations.py => migrations/_legacy.py} (91%) create mode 100644 src/vouch/migrations/journal.py create mode 100644 src/vouch/migrations/manifest.py create mode 100644 src/vouch/migrations/rewriter.py create mode 100644 src/vouch/migrations/runner.py create mode 100644 src/vouch/migrations/schema.py create mode 100644 src/vouch/migrations/semver.py create mode 100644 tests/fixtures/migrations/0099-test-rename.yaml create mode 100644 tests/test_schema_migrations.py diff --git a/docs/migrations.md b/docs/migrations.md new file mode 100644 index 00000000..79627f15 --- /dev/null +++ b/docs/migrations.md @@ -0,0 +1,75 @@ +# Schema migrations — `vouch migrate` + +The `.vouch/` layout is the durable contract: yaml claims, markdown pages with +frontmatter, json sessions, the jsonl audit log. As the pydantic models in +`src/vouch/models.py` evolve, a KB created against an older model would become a +load-time error. `vouch migrate` gives that evolution a versioned, reversible, +audit-logged upgrade path so a schema change never silently breaks a KB in the +wild. + +## Two version axes + +vouch tracks two independent versions, and `vouch migrate` covers both: + +| | Stamp | What it governs | Reached by | +|---|---|---|---| +| **Format** (integer) | `config.yaml` `version` | the `.vouch/` directory layout (subdirs, `.gitignore`) | `vouch migrate` *(no subcommand)* | +| **Schema** (semver) | `.vouch/schema_version` | the model schema of each artifact | `vouch migrate ` | + +A KB with no `.vouch/schema_version` file is treated as the baseline (`0.1.0`), +so existing KBs keep loading until their first migrate. `vouch init` stamps the +current version on bootstrap. + +## Commands + +```bash +vouch migrate status # current schema version, target, pending migrations +vouch migrate plan # dry-run: every file each pending migration would change +vouch migrate plan --to 0.3.0 # plan against a specific target +vouch migrate apply # apply pending migrations (audit-logged, atomic) +vouch migrate apply --yes # skip the confirmation prompt (CI) +vouch migrate rollback # reverse the most recently applied migration +vouch migrate verify # parse-load every artifact under the current version +``` + +## Manifests + +Migrations are **data, not code**: yaml manifests in the repo-root `migrations/` +directory, one consecutive version step each. See +[`migrations/README.md`](../migrations/README.md) for the full format and the +transform verbs (`rename`, `default`, `drop`, `split`, `merge`). Only consecutive +manifests apply — a `0.1 → 0.5` KB walks through `0.2`, `0.3`, `0.4` in order. + +## Safety model + +- **Atomic per file.** Each artifact is rewritten to a temp file, `fsync`-ed, + then `os.replace`-d into place. A file is always its old bytes or its new + bytes — never a torn mix. +- **Reversible.** Before any rewrite, the prior content of every file the step + touches is journalled to `.vouch/migrations/rollback-.jsonl`. `vouch + migrate rollback` replays it, restoring the exact prior bytes — so + apply → rollback is byte-equivalent (modulo the audit-log entries recording + the round trip). +- **Crash-safe.** The journal is written and fsynced *before* the first rewrite, + and `.vouch/schema_version` is bumped *last*. An interrupted apply therefore + leaves the KB reporting its prior version, with a journal to recover from. +- **Precondition.** Apply refuses if the KB has pending proposals — it won't + rewrite a reviewer's in-flight queue out from under them. Resolve them with + `vouch list-pending` first. +- **`state.db` is disposable.** The runner does not migrate the FTS5 / embedding + cache; the CLI rebuilds it after a successful apply. + +## Audit trail + +Each applied manifest logs one `kb.migrate.apply` event (manifest id, file count, +rollback-journal id); a rollback logs `kb.migrate.rollback`. The legacy format +migration continues to log `kb.migrate`. + +## Out of scope (v1) + +- Cross-major jumps in a single manifest (consecutive only). +- `custom: path/to/migration.py` script transforms (lands once the verb + taxonomy stabilises). +- Migrating `state.db` (disposable; rebuilt by `vouch index`). +- Bundle version embedding / refusing mismatched imports — a follow-up. +- Concurrent multi-process migrate runs (single-writer assumption). diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 00000000..ba77d32e --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,57 @@ +# Schema migration manifests + +Each file here is a **migration manifest**: a single, consecutive +model-schema version step, declared as data. Adding one is a contributor-friendly, +single-file PR — the same shape as `adapters/`. The runner +(`vouch.migrations.runner`) chains them from a KB's current +`.vouch/schema_version` toward the target. + +Provenance is derived; migrations are not. A manifest rewrites the durable +source-of-truth files under `.vouch/` (yaml claims, markdown page frontmatter, +etc.). The `state.db` cache is disposable and rebuilt afterward by `vouch index`. + +## File naming + +`NNNN-short-slug.yaml`, e.g. `0001-add-scope-spec.yaml`. Files are applied in +filename order; exactly one manifest may migrate *from* any given version. + +## Format + +```yaml +from_version: "0.1.0" # must equal the previous manifest's to_version +to_version: "0.2.0" # strictly greater than from_version +artifact: claims # claims | pages | entities | relations | evidence | sessions +description: "rename confidence -> certainty; default scope=project" +transforms: # applied in order to each artifact's dict + - rename: {from: confidence, to: certainty} + - default: {field: scope, value: project} + - drop: {field: legacy_flag} + - split: {field: name, into: [first, last], on: " "} + - merge: {fields: [first, last], into: name, with: " "} +reverse: # documents the inverse (rollback is journal-based) + - rename: {from: certainty, to: confidence} + - drop: {field: scope} +``` + +### Transform verbs + +| Verb | Spec | Effect | +|------|------|--------| +| `rename` | `{from, to}` | move a field's value to a new key | +| `default` | `{field, value}` | set a field only if absent | +| `drop` | `{field}` | remove a field | +| `split` | `{field, into: [..], on}` | split a string field into several | +| `merge` | `{fields: [..], into, with}` | join several fields into one | + +## Apply / rollback + +`vouch migrate apply` rewrites each artifact atomically (temp + fsync + +rename) and journals the prior content to `.vouch/migrations/rollback-.jsonl` +*before* writing, then bumps `.vouch/schema_version` last. `vouch migrate +rollback` replays the newest journal to restore the exact prior bytes. An +interrupted apply therefore always leaves the KB at its prior version with a +journal to recover from. + +> No real manifests ship yet — the first lands with ROADMAP 0.2's multi-dim +> scopes (#2). This directory documents the convention and is exercised by the +> synthetic manifests in `tests/fixtures/migrations/`. diff --git a/src/vouch/cli.py b/src/vouch/cli.py index 98e562c4..11b8eac0 100644 --- a/src/vouch/cli.py +++ b/src/vouch/cli.py @@ -374,18 +374,28 @@ def fsck() -> None: sys.exit(0 if report.ok else 1) -@cli.command() -@click.option("--check", "check_only", is_flag=True, help="Only check whether migration is needed.") -@click.option("--dry-run", is_flag=True, help="Show planned changes without writing.") -@click.option("--to-version", type=int, default=None, help="Target KB format version.") +@cli.group(invoke_without_command=True) +@click.option("--check", "check_only", is_flag=True, help="Only check if a migration is needed.") +@click.option("--dry-run", is_flag=True, help="Show planned format changes without writing.") +@click.option("--to-version", type=int, default=None, help="Target KB format (integer) version.") @click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +@click.pass_context def migrate( + ctx: click.Context, check_only: bool, dry_run: bool, to_version: int | None, as_json: bool, ) -> None: - """Upgrade the on-disk .vouch/ layout to the supported format.""" + """Migrate the KB. + + With no subcommand, runs the legacy integer *format* migration of the + .vouch/ directory layout. The subcommands (status / plan / apply / rollback + / verify) drive the semver model-schema migrations keyed off + .vouch/schema_version. + """ + if ctx.invoked_subcommand is not None: + return if check_only and dry_run: raise click.ClickException("--check and --dry-run are mutually exclusive") @@ -430,6 +440,128 @@ def migrate( sys.exit(1) +@migrate.command("status") +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def migrate_status(as_json: bool) -> None: + """Show the KB schema version, target, and pending migrations.""" + store = _load_store() + with _cli_errors(): + result = migrations_mod.schema_status(store) + if as_json: + _emit_json(result) + return + click.echo(f"schema: {result['schema_version']} -> {result['target_version']}") + if result["up_to_date"]: + click.echo("up to date") + else: + click.echo(f"pending: {', '.join(result['pending'])}") + + +@migrate.command("plan") +@click.option("--to", "to_version", default=None, help="Target schema version (semver).") +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def migrate_plan(to_version: str | None, as_json: bool) -> None: + """Dry-run: list every file each pending migration would change.""" + store = _load_store() + with _cli_errors(): + result = migrations_mod.schema_plan(store, to_version=to_version) + if as_json: + _emit_json(result) + return + if not result["needed"]: + click.echo(f"schema: {result['current_version']} (up to date)") + return + click.echo(f"schema: {result['current_version']} -> {result['target_version']}") + for step in result["steps"]: + click.echo( + f"- {step['manifest_id']}: {step['from_version']} -> {step['to_version']} " + f"({step['artifact']}, {step['file_count']} file(s))" + ) + for rel in step["changed"]: + click.echo(f" * {rel}") + click.echo(f"total: {result['total_files']} file(s)") + + +@migrate.command("apply") +@click.option("--to", "to_version", default=None, help="Target schema version (semver).") +@click.option("--yes", is_flag=True, help="Skip the confirmation prompt (CI).") +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def migrate_apply(to_version: str | None, yes: bool, as_json: bool) -> None: + """Apply pending schema migrations (audit-logged, atomic, reversible).""" + store = _load_store() + with _cli_errors(): + preview = migrations_mod.schema_plan(store, to_version=to_version) + if not preview["needed"]: + if as_json: + _emit_json( + { + "applied": False, + "from_version": preview["current_version"], + "to_version": preview["current_version"], + "manifests": [], + "files": 0, + } + ) + else: + click.echo(f"schema: {preview['current_version']} (up to date)") + return + if not yes: + click.confirm( + f"Apply {len(preview['steps'])} migration(s) " + f"({preview['current_version']} -> {preview['target_version']}, " + f"{preview['total_files']} file(s))?", + abort=True, + ) + result = migrations_mod.schema_apply(store, to_version=to_version, actor=_whoami()) + # state.db is derived; rebuild it under the new layout. + health.rebuild_index(store) + if as_json: + _emit_json(result) + else: + click.echo( + f"schema: {result['from_version']} -> {result['to_version']} " + f"({result['files']} file(s), {len(result['manifests'])} manifest(s))" + ) + + +@migrate.command("rollback") +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def migrate_rollback(as_json: bool) -> None: + """Reverse the most recently applied schema migration.""" + store = _load_store() + with _cli_errors(): + result = migrations_mod.schema_rollback(store, actor=_whoami()) + health.rebuild_index(store) + if as_json: + _emit_json(result) + else: + click.echo( + f"schema: {result['from_version']} -> {result['to_version']} " + f"(rolled back {result['files']} file(s))" + ) + + +@migrate.command("verify") +@click.option("--json", "as_json", is_flag=True, help="Emit JSON instead of text.") +def migrate_verify(as_json: bool) -> None: + """Parse-load every artifact under the current schema version.""" + store = _load_store() + with _cli_errors(): + result = migrations_mod.schema_verify(store) + if as_json: + _emit_json(result) + elif result["ok"]: + click.echo( + f"verified {result['checked']} artifact(s) at schema {result['schema_version']}" + ) + else: + click.echo(f"FAILED: {len(result['errors'])} of {result['checked']} artifact(s)") + for err in result["errors"]: + click.echo(f" ✗ {err['path']}: {err['error']}") + if not result["ok"]: + sys.exit(1) + + # --- metrics -------------------------------------------------------------- diff --git a/src/vouch/migrations/__init__.py b/src/vouch/migrations/__init__.py new file mode 100644 index 00000000..43ae3d96 --- /dev/null +++ b/src/vouch/migrations/__init__.py @@ -0,0 +1,78 @@ +"""KB migrations. + +Two layers live here, by design: + +* **Legacy integer "format" migrations** (`_legacy`) — the ``config.yaml`` + ``version`` / ``KB_FORMAT_VERSION`` directory-layout bump, reached by + ``vouch migrate`` with no subcommand. Preserved verbatim. +* **Semver model-schema migrations** (`runner` + `manifest` + `rewriter` + + `journal` + `schema`) — the versioned plan/apply/rollback/verify flow keyed off + ``.vouch/schema_version`` and driven by yaml manifests. Reached by + ``vouch migrate status|plan|apply|rollback|verify``. + +Both are re-exported here so importers keep using ``vouch.migrations`` unchanged. +""" + +from __future__ import annotations + +from ._legacy import ( + MigrationError, + MigrationPlan, + MigrationResult, + MigrationStep, + build_plan, + detect_version, + migrate, + read_config, + write_config, +) +from .manifest import ( + Manifest, + ManifestError, + default_manifests_dir, + load_manifests, + parse_manifest, +) +from .runner import ( + SchemaPlan, + SchemaPlanStep, + build_schema_plan, +) +from .runner import apply as schema_apply +from .runner import plan as schema_plan +from .runner import rollback as schema_rollback +from .runner import status as schema_status +from .runner import verify as schema_verify +from .schema import ( + BASELINE_SCHEMA_VERSION, + read_schema_version, + write_schema_version, +) + +__all__ = [ + "BASELINE_SCHEMA_VERSION", + "Manifest", + "ManifestError", + "MigrationError", + "MigrationPlan", + "MigrationResult", + "MigrationStep", + "SchemaPlan", + "SchemaPlanStep", + "build_plan", + "build_schema_plan", + "default_manifests_dir", + "detect_version", + "load_manifests", + "migrate", + "parse_manifest", + "read_config", + "read_schema_version", + "schema_apply", + "schema_plan", + "schema_rollback", + "schema_status", + "schema_verify", + "write_config", + "write_schema_version", +] diff --git a/src/vouch/migrations.py b/src/vouch/migrations/_legacy.py similarity index 91% rename from src/vouch/migrations.py rename to src/vouch/migrations/_legacy.py index bdb70375..426b5ac9 100644 --- a/src/vouch/migrations.py +++ b/src/vouch/migrations/_legacy.py @@ -1,8 +1,10 @@ -"""On-disk KB format migrations. +"""Legacy integer "KB format" migrations. -The migration layer only changes source-of-truth files under `.vouch/`. -Derived caches such as state.db are rebuilt by the CLI after a successful -apply. +This predates the semver model-schema runner (see :mod:`.runner`) and governs the +*directory layout* version stamped in ``config.yaml`` (``KB_FORMAT_VERSION``) — +making sure the ``.vouch/`` subdir tree and ``.gitignore`` exist. It is reached +by ``vouch migrate`` with no subcommand and is preserved verbatim; the semver +runner is a separate, additive concern keyed off ``.vouch/schema_version``. """ from __future__ import annotations @@ -13,7 +15,7 @@ import yaml -from .storage import KB_FORMAT_VERSION, SUBDIRS, KBStore, _starter_config +from ..storage import KB_FORMAT_VERSION, SUBDIRS, KBStore, _starter_config class MigrationError(RuntimeError): diff --git a/src/vouch/migrations/journal.py b/src/vouch/migrations/journal.py new file mode 100644 index 00000000..13d28e53 --- /dev/null +++ b/src/vouch/migrations/journal.py @@ -0,0 +1,71 @@ +"""The rollback journal. + +Before a manifest rewrites any artifact, the *prior* content of every file it +will touch is captured to ``.vouch/migrations/rollback-.jsonl`` (header line ++ one line per file). Rollback replays it to restore those exact bytes, which is +what makes ``apply`` → ``rollback`` byte-equivalent regardless of how lossy the +forward transform was. The journal is written and fsynced *before* the first +rewrite, so an interrupted apply always leaves a journal to recover from. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +from ..storage import KBStore +from .rewriter import atomic_write_text + +MIGRATIONS_SUBDIR = "migrations" # under .vouch/ + + +@dataclass(frozen=True) +class JournalEntry: + rel_path: str + before: str | None # None => the file did not exist before (rollback deletes it) + + +def journal_dir(store: KBStore) -> Path: + return store.kb_dir / MIGRATIONS_SUBDIR + + +def journal_path(store: KBStore, journal_id: str) -> Path: + return journal_dir(store) / f"rollback-{journal_id}.jsonl" + + +def write_journal( + store: KBStore, journal_id: str, header: dict[str, object], entries: list[JournalEntry] +) -> Path: + path = journal_path(store, journal_id) + lines = [json.dumps({"_header": header}, sort_keys=True)] + for e in entries: + lines.append(json.dumps({"path": e.rel_path, "before": e.before}, sort_keys=True)) + atomic_write_text(path, "\n".join(lines) + "\n") + return path + + +def list_journals(store: KBStore) -> list[Path]: + d = journal_dir(store) + if not d.is_dir(): + return [] + return sorted(d.glob("rollback-*.jsonl")) + + +def latest_journal(store: KBStore) -> Path | None: + journals = list_journals(store) + return journals[-1] if journals else None + + +def read_journal(path: Path) -> tuple[dict[str, object], list[JournalEntry]]: + header: dict[str, object] = {} + entries: list[JournalEntry] = [] + for line in path.read_text().splitlines(): + if not line.strip(): + continue + rec = json.loads(line) + if "_header" in rec: + header = rec["_header"] + else: + entries.append(JournalEntry(rel_path=rec["path"], before=rec["before"])) + return header, entries diff --git a/src/vouch/migrations/manifest.py b/src/vouch/migrations/manifest.py new file mode 100644 index 00000000..b67ddda5 --- /dev/null +++ b/src/vouch/migrations/manifest.py @@ -0,0 +1,152 @@ +"""Migration manifests — first-class yaml artifacts. + +Each manifest declares a single consecutive version step for one artifact kind: + + from_version: "0.1.0" + to_version: "0.2.0" + artifact: claims + description: "rename confidence -> certainty" + transforms: + - rename: {from: confidence, to: certainty} + - default: {field: scope, value: project} + reverse: + - rename: {from: certainty, to: confidence} + +Manifests live in a repo-root ``migrations/`` directory (additive, +contributor-friendly, single-file PRs — same shape as ``adapters/``). The +``reverse`` block documents the inverse; rollback itself is content-based via the +journal, so it stays exact regardless. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from . import semver +from .rewriter import ARTIFACT_KINDS, apply_transforms + +VERBS = {"rename", "default", "drop", "split", "merge"} + +#: Env override for the manifest directory (mainly for tests / vendored layouts). +MANIFESTS_DIR_ENV = "VOUCH_MIGRATIONS_DIR" + + +class ManifestError(RuntimeError): + """A manifest file is malformed or the manifest set is inconsistent.""" + + +@dataclass(frozen=True) +class Manifest: + manifest_id: str # filename stem, e.g. "0001-add-scope-spec" + from_version: str + to_version: str + description: str + artifact: str + transforms: list[dict[str, Any]] + reverse: list[dict[str, Any]] + path: Path + + +def _validate_transforms(transforms: Any, where: str) -> list[dict[str, Any]]: + if not isinstance(transforms, list): + raise ManifestError(f"{where}: transforms must be a list") + out: list[dict[str, Any]] = [] + for t in transforms: + if not isinstance(t, dict) or len(t) != 1: + raise ManifestError(f"{where}: each transform is a single-key mapping {{verb: spec}}") + ((verb, _spec),) = t.items() + if verb not in VERBS: + raise ManifestError( + f"{where}: unknown transform verb {verb!r} (allowed: {sorted(VERBS)})" + ) + out.append(t) + return out + + +def parse_manifest(path: Path) -> Manifest: + try: + data = yaml.safe_load(path.read_text()) + except yaml.YAMLError as e: + raise ManifestError(f"{path.name}: invalid yaml: {e}") from e + if not isinstance(data, dict): + raise ManifestError(f"{path.name}: manifest must be a mapping") + for key in ("from_version", "to_version", "artifact"): + if key not in data: + raise ManifestError(f"{path.name}: missing required key {key!r}") + from_v, to_v = str(data["from_version"]), str(data["to_version"]) + if not semver.is_valid(from_v) or not semver.is_valid(to_v): + raise ManifestError(f"{path.name}: from_version/to_version must be MAJOR.MINOR.PATCH") + if not semver.lt(from_v, to_v): + raise ManifestError(f"{path.name}: to_version must be greater than from_version") + artifact = str(data["artifact"]) + if artifact not in ARTIFACT_KINDS: + raise ManifestError( + f"{path.name}: unknown artifact {artifact!r} (allowed: {sorted(ARTIFACT_KINDS)})" + ) + transforms = _validate_transforms(data.get("transforms", []), path.name) + reverse = _validate_transforms(data.get("reverse", []), path.name) + return Manifest( + manifest_id=path.stem, + from_version=from_v, + to_version=to_v, + description=str(data.get("description", "")), + artifact=artifact, + transforms=transforms, + reverse=reverse, + path=path, + ) + + +def load_manifests(manifests_dir: Path | None) -> list[Manifest]: + """Parse every ``*.yaml`` manifest in the directory, sorted by filename.""" + if manifests_dir is None or not manifests_dir.is_dir(): + return [] + out = [parse_manifest(p) for p in sorted(manifests_dir.glob("*.yaml"))] + _check_no_duplicate_steps(out) + return out + + +def _check_no_duplicate_steps(manifests: list[Manifest]) -> None: + seen: dict[str, str] = {} + for m in manifests: + if m.from_version in seen: + raise ManifestError( + f"two manifests migrate from {m.from_version}: " + f"{seen[m.from_version]} and {m.manifest_id}" + ) + seen[m.from_version] = m.manifest_id + + +def default_manifests_dir() -> Path | None: + """Resolve the manifest directory: env override, else repo-root ``migrations/``. + + Walks up from this package looking for a directory that holds both + ``pyproject.toml`` and a ``migrations/`` subdir (a source checkout). Returns + ``None`` when neither is found (e.g. an installed wheel with no manifests yet). + """ + env = os.environ.get(MANIFESTS_DIR_ENV) + if env: + return Path(env) + here = Path(__file__).resolve() + for parent in here.parents: + candidate = parent / "migrations" + if (parent / "pyproject.toml").is_file() and candidate.is_dir(): + return candidate + return None + + +# Re-exported so callers don't need a separate import to preview a transform. +__all__ = [ + "VERBS", + "Manifest", + "ManifestError", + "apply_transforms", + "default_manifests_dir", + "load_manifests", + "parse_manifest", +] diff --git a/src/vouch/migrations/rewriter.py b/src/vouch/migrations/rewriter.py new file mode 100644 index 00000000..d3dd5d06 --- /dev/null +++ b/src/vouch/migrations/rewriter.py @@ -0,0 +1,135 @@ +"""Atomic file rewrites and the manifest transform verbs. + +Every mutation a migration performs goes through :func:`atomic_write_text`: +write to a temp file in the same directory, ``fsync``, then ``os.replace`` (an +atomic rename on POSIX). A crash therefore never leaves a half-written artifact — +a file is either its old bytes or its new bytes, never a torn mix. This mirrors +the audit log's own fsync-on-append durability; ``KBStore`` exposes no shared +atomic-write helper, so the migration layer provides its own. +""" + +from __future__ import annotations + +import contextlib +import os +import tempfile +from pathlib import Path + +from ..storage import _FRONTMATTER_RE, _yaml_dump, _yaml_load + +# Artifact kinds a manifest may target, mapped to (subdir, on-disk format). +ARTIFACT_KINDS: dict[str, tuple[str, str]] = { + "claims": ("claims", "yaml"), + "entities": ("entities", "yaml"), + "relations": ("relations", "yaml"), + "evidence": ("evidence", "yaml"), + "sessions": ("sessions", "yaml"), + "pages": ("pages", "md"), +} + + +def atomic_write_text(path: Path, text: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=".mig-", suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(text) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + except BaseException: + # Leave no temp files behind on failure (incl. KeyboardInterrupt). + with contextlib.suppress(FileNotFoundError): + os.unlink(tmp) + raise + + +# --- transform verbs ------------------------------------------------------- + + +def _t_rename(d: dict, spec: dict) -> None: + frm, to = spec["from"], spec["to"] + if frm in d: + d[to] = d.pop(frm) + + +def _t_default(d: dict, spec: dict) -> None: + field = spec["field"] + if field not in d: + d[field] = spec["value"] + + +def _t_drop(d: dict, spec: dict) -> None: + d.pop(spec["field"], None) + + +def _t_split(d: dict, spec: dict) -> None: + field = spec["field"] + into = list(spec["into"]) + sep = spec.get("on", " ") + if field not in d: + return + parts = str(d[field]).split(sep) + for i, name in enumerate(into): + d[name] = parts[i] if i < len(parts) else "" + if field not in into: + del d[field] + + +def _t_merge(d: dict, spec: dict) -> None: + fields = list(spec["fields"]) + into = spec["into"] + joiner = spec.get("with", " ") + values = [str(d.get(f, "")) for f in fields] + d[into] = joiner.join(values) + for f in fields: + if f != into: + d.pop(f, None) + + +_VERB_FNS = { + "rename": _t_rename, + "default": _t_default, + "drop": _t_drop, + "split": _t_split, + "merge": _t_merge, +} + + +def apply_transforms(data: dict, transforms: list[dict]) -> dict: + """Return a new dict with each transform applied in order.""" + out = dict(data) + for transform in transforms: + # Each transform is a single-key mapping: {verb: spec}. + ((verb, spec),) = transform.items() + _VERB_FNS[verb](out, spec) + return out + + +# --- per-file content transforms ------------------------------------------ + + +def artifact_files(kb_dir: Path, kind: str) -> list[Path]: + subdir, fmt = ARTIFACT_KINDS[kind] + ext = "*.md" if fmt == "md" else "*.yaml" + return sorted((kb_dir / subdir).glob(ext)) + + +def transform_text(text: str, kind: str, transforms: list[dict]) -> str: + """Apply a manifest's transforms to one artifact file's text.""" + fmt = ARTIFACT_KINDS[kind][1] + if fmt == "yaml": + data = _yaml_load(text) + if not isinstance(data, dict): + return text + return _yaml_dump(apply_transforms(data, transforms)) + # markdown page: transform the YAML frontmatter, leave the body untouched. + match = _FRONTMATTER_RE.match(text) + if not match: + return text + front = _yaml_load(match.group(1)) or {} + if not isinstance(front, dict): + return text + body = match.group(2) + new_front = _yaml_dump(apply_transforms(front, transforms)).rstrip("\n") + return f"---\n{new_front}\n---\n{body}" diff --git a/src/vouch/migrations/runner.py b/src/vouch/migrations/runner.py new file mode 100644 index 00000000..1192fead --- /dev/null +++ b/src/vouch/migrations/runner.py @@ -0,0 +1,336 @@ +"""The semver model-schema migration runner: status / plan / apply / rollback / +verify. + +The runner walks *consecutive* manifests from the KB's current +``.vouch/schema_version`` toward a target, rewriting one artifact kind per +manifest atomically and journalling the prior content so the step is reversible. +The schema-version stamp is bumped **last**, so an interrupted apply leaves the +KB reporting its prior version with a journal available for rollback. +""" + +from __future__ import annotations + +import time +import uuid +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from pydantic import BaseModel + +from .. import audit +from ..models import ( + Claim, + Entity, + Evidence, + Page, + Relation, + Session, + Source, +) +from ..models import ( + ProposalStatus as _ProposalStatus, +) +from ..storage import _FRONTMATTER_RE, KBStore, _yaml_load +from . import journal as journal_mod +from . import schema, semver +from ._legacy import MigrationError +from .journal import JournalEntry +from .manifest import Manifest, default_manifests_dir, load_manifests +from .rewriter import artifact_files, atomic_write_text, transform_text + + +class CrashSimulated(RuntimeError): + """Raised by the ``_fail_after`` test hook to emulate a mid-apply crash.""" + + +@dataclass(frozen=True) +class SchemaPlanStep: + manifest_id: str + from_version: str + to_version: str + artifact: str + description: str + changed: list[str] = field(default_factory=list) + + @property + def file_count(self) -> int: + return len(self.changed) + + +@dataclass(frozen=True) +class SchemaPlan: + current_version: str + target_version: str + steps: list[SchemaPlanStep] + + @property + def needed(self) -> bool: + return bool(self.steps) + + +def _resolve_dir(manifests_dir: Path | None) -> Path | None: + return manifests_dir if manifests_dir is not None else default_manifests_dir() + + +def _by_from(manifests: list[Manifest]) -> dict[str, Manifest]: + return {m.from_version: m for m in manifests} + + +def _latest_reachable(by_from: dict[str, Manifest], current: str) -> str: + version = current + while version in by_from: + version = by_from[version].to_version + return version + + +def _changed_files(store: KBStore, manifest: Manifest) -> list[tuple[Path, str, str]]: + """Files whose content the manifest would actually change: (path, old, new).""" + out: list[tuple[Path, str, str]] = [] + for path in artifact_files(store.kb_dir, manifest.artifact): + old = path.read_text() + new = transform_text(old, manifest.artifact, manifest.transforms) + if new != old: + out.append((path, old, new)) + return out + + +def build_schema_plan( + store: KBStore, *, to_version: str | None = None, manifests_dir: Path | None = None +) -> SchemaPlan: + current = schema.read_schema_version(store) + manifests = load_manifests(_resolve_dir(manifests_dir)) + by_from = _by_from(manifests) + target = to_version if to_version is not None else _latest_reachable(by_from, current) + if not semver.is_valid(target): + raise MigrationError(f"invalid target schema version {target!r}") + if semver.lt(target, current): + raise MigrationError( + f"cannot migrate backwards from schema version {current} to {target}" + ) + + steps: list[SchemaPlanStep] = [] + version = current + while version != target: + manifest = by_from.get(version) + if manifest is None: + raise MigrationError(f"no migration registered from schema version {version}") + if semver.lt(target, manifest.to_version): + raise MigrationError( + f"no manifest stops at {target}; " + f"{manifest.manifest_id} jumps {version} -> {manifest.to_version}" + ) + changed = [str(p.relative_to(store.kb_dir)) for p, _, _ in _changed_files(store, manifest)] + steps.append( + SchemaPlanStep( + manifest_id=manifest.manifest_id, + from_version=manifest.from_version, + to_version=manifest.to_version, + artifact=manifest.artifact, + description=manifest.description, + changed=sorted(changed), + ) + ) + version = manifest.to_version + return SchemaPlan(current_version=current, target_version=target, steps=steps) + + +def status(store: KBStore, *, manifests_dir: Path | None = None) -> dict[str, Any]: + current = schema.read_schema_version(store) + manifests = load_manifests(_resolve_dir(manifests_dir)) + by_from = _by_from(manifests) + target = _latest_reachable(by_from, current) + pending: list[str] = [] + version = current + while version in by_from and version != target: + pending.append(by_from[version].manifest_id) + version = by_from[version].to_version + journals = [p.name for p in journal_mod.list_journals(store)] + return { + "schema_version": current, + "target_version": target, + "up_to_date": current == target, + "pending": pending, + "applied_journals": journals, + } + + +def plan( + store: KBStore, *, to_version: str | None = None, manifests_dir: Path | None = None +) -> dict[str, Any]: + sp = build_schema_plan(store, to_version=to_version, manifests_dir=manifests_dir) + return { + "schema_version": 1, + "current_version": sp.current_version, + "target_version": sp.target_version, + "needed": sp.needed, + "total_files": sum(s.file_count for s in sp.steps), + "steps": [{**asdict(s), "file_count": s.file_count} for s in sp.steps], + } + + +def _new_journal_id(index: int) -> str: + ts = time.strftime("%Y%m%d-%H%M%S", time.gmtime()) + return f"{ts}-{index:03d}-{uuid.uuid4().hex[:8]}" + + +def _pending_proposals_guard(store: KBStore) -> None: + pending = store.list_proposals(_ProposalStatus.PENDING) + if pending: + raise MigrationError( + f"KB has {len(pending)} pending proposal(s); " + "run `vouch list-pending` and resolve before migrating" + ) + + +def apply( + store: KBStore, + *, + to_version: str | None = None, + manifests_dir: Path | None = None, + actor: str = "vouch", + _fail_after: int | None = None, +) -> dict[str, Any]: + _pending_proposals_guard(store) + sp = build_schema_plan(store, to_version=to_version, manifests_dir=manifests_dir) + if not sp.needed: + return { + "applied": False, + "from_version": sp.current_version, + "to_version": sp.current_version, + "manifests": [], + "files": 0, + } + manifests = {m.manifest_id: m for m in load_manifests(_resolve_dir(manifests_dir))} + applied: list[str] = [] + total = 0 + written = 0 + for index, step in enumerate(sp.steps): + manifest = manifests[step.manifest_id] + changes = _changed_files(store, manifest) + journal_id = _new_journal_id(index) + entries = [ + JournalEntry(rel_path=str(path.relative_to(store.kb_dir)), before=old) + for path, old, _ in changes + ] + # Journal first (fsynced) so an interrupted apply is always recoverable. + journal_mod.write_journal( + store, + journal_id, + { + "manifest": manifest.manifest_id, + "from": manifest.from_version, + "to": manifest.to_version, + }, + entries, + ) + for path, _old, new in changes: + if _fail_after is not None and written >= _fail_after: + raise CrashSimulated(f"simulated crash after {written} file(s)") + atomic_write_text(path, new) + written += 1 + # Version stamp bumped last: a crash above leaves the prior version. + schema.write_schema_version(store, manifest.to_version) + audit.log_event( + store.kb_dir, + event="kb.migrate.apply", + actor=actor, + object_ids=[journal_id], + reversible=True, + data={ + "manifest": manifest.manifest_id, + "from": manifest.from_version, + "to": manifest.to_version, + "files": len(changes), + "journal": journal_id, + }, + ) + applied.append(manifest.manifest_id) + total += len(changes) + return { + "applied": True, + "from_version": sp.current_version, + "to_version": sp.target_version, + "manifests": applied, + "files": total, + } + + +def rollback(store: KBStore, *, actor: str = "vouch") -> dict[str, Any]: + path = journal_mod.latest_journal(store) + if path is None: + raise MigrationError("no applied migration to roll back") + header, entries = journal_mod.read_journal(path) + for entry in entries: + target = store.kb_dir / entry.rel_path + if entry.before is None: + target.unlink(missing_ok=True) + else: + atomic_write_text(target, entry.before) + from_version = str(header.get("from", schema.BASELINE_SCHEMA_VERSION)) + to_version = str(header.get("to", "")) + schema.write_schema_version(store, from_version) + path.unlink() + audit.log_event( + store.kb_dir, + event="kb.migrate.rollback", + actor=actor, + object_ids=[str(header.get("manifest", ""))], + reversible=False, + data={ + "manifest": header.get("manifest"), + "from": to_version, + "to": from_version, + "files": len(entries), + }, + ) + return { + "rolled_back": True, + "from_version": to_version, + "to_version": from_version, + "manifest": header.get("manifest"), + "files": len(entries), + } + + +# artifact kind -> (subdir, pydantic model) for verify's per-file load check. +_YAML_MODELS: dict[str, type[BaseModel]] = { + "claims": Claim, + "entities": Entity, + "relations": Relation, + "evidence": Evidence, + "sessions": Session, +} + + +def verify(store: KBStore) -> dict[str, Any]: + """Parse-load every artifact under the current version; collect any failures.""" + errors: list[dict[str, str]] = [] + checked = 0 + for kind, model in _YAML_MODELS.items(): + for path in artifact_files(store.kb_dir, kind): + checked += 1 + try: + model.model_validate(_yaml_load(path.read_text())) + except Exception as e: + errors.append({"path": str(path.relative_to(store.kb_dir)), "error": str(e)}) + for path in artifact_files(store.kb_dir, "pages"): + checked += 1 + try: + match = _FRONTMATTER_RE.match(path.read_text()) + front = _yaml_load(match.group(1)) if match else {} + Page.model_validate({**(front or {}), "body": match.group(2) if match else ""}) + except Exception as e: + errors.append({"path": str(path.relative_to(store.kb_dir)), "error": str(e)}) + for meta in sorted((store.kb_dir / "sources").glob("*/meta.yaml")): + checked += 1 + try: + Source.model_validate(_yaml_load(meta.read_text())) + except Exception as e: + errors.append({"path": str(meta.relative_to(store.kb_dir)), "error": str(e)}) + return { + "schema_version": schema.read_schema_version(store), + "checked": checked, + "ok": not errors, + "errors": errors, + } diff --git a/src/vouch/migrations/schema.py b/src/vouch/migrations/schema.py new file mode 100644 index 00000000..7ef9681c --- /dev/null +++ b/src/vouch/migrations/schema.py @@ -0,0 +1,37 @@ +"""The on-disk schema-version stamp (``.vouch/schema_version``). + +A single-line semver. Absent means "treat as the baseline" so KBs created before +this feature keep loading until their first migrate. +""" + +from __future__ import annotations + +from pathlib import Path + +from ..storage import SCHEMA_VERSION as _CURRENT +from ..storage import SCHEMA_VERSION_FILENAME, KBStore +from . import semver +from .rewriter import atomic_write_text + +#: KBs with no `schema_version` file are treated as this version. +BASELINE_SCHEMA_VERSION = _CURRENT + + +def schema_version_path(store: KBStore) -> Path: + return store.kb_dir / SCHEMA_VERSION_FILENAME + + +def read_schema_version(store: KBStore) -> str: + p = schema_version_path(store) + if not p.exists(): + return BASELINE_SCHEMA_VERSION + raw = p.read_text().strip() + if not raw: + return BASELINE_SCHEMA_VERSION + semver.parse(raw) # validate; raises ValueError on garbage + return raw + + +def write_schema_version(store: KBStore, version: str) -> None: + semver.parse(version) + atomic_write_text(schema_version_path(store), version + "\n") diff --git a/src/vouch/migrations/semver.py b/src/vouch/migrations/semver.py new file mode 100644 index 00000000..2f174684 --- /dev/null +++ b/src/vouch/migrations/semver.py @@ -0,0 +1,37 @@ +"""A deliberately tiny semver subset (``MAJOR.MINOR.PATCH``, no pre-release tags). + +Schema versions are short and fully controlled by manifests in-repo, so a 30-line +parser beats a dependency. Parsed versions are plain int tuples and compare with +the usual operators. +""" + +from __future__ import annotations + +import re + +_SEMVER_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") + +Version = tuple[int, int, int] + + +def parse(value: str) -> Version: + m = _SEMVER_RE.match(value.strip()) + if not m: + raise ValueError(f"invalid schema version {value!r} (expected MAJOR.MINOR.PATCH)") + return (int(m.group(1)), int(m.group(2)), int(m.group(3))) + + +def is_valid(value: str) -> bool: + return bool(_SEMVER_RE.match(value.strip())) + + +def lt(a: str, b: str) -> bool: + return parse(a) < parse(b) + + +def le(a: str, b: str) -> bool: + return parse(a) <= parse(b) + + +def eq(a: str, b: str) -> bool: + return parse(a) == parse(b) diff --git a/src/vouch/storage.py b/src/vouch/storage.py index 4519e777..88c600cc 100644 --- a/src/vouch/storage.py +++ b/src/vouch/storage.py @@ -52,6 +52,12 @@ CONFIG_FILENAME = "config.yaml" KB_FORMAT_VERSION = 1 +# Semver model-schema version stamped on bootstrap; the migration runner +# (vouch.migrations) advances it. Distinct from KB_FORMAT_VERSION, which is the +# integer directory-layout version in config.yaml. +SCHEMA_VERSION = "0.1.0" +SCHEMA_VERSION_FILENAME = "schema_version" + SUBDIRS = ( "claims", "pages", "sources", "entities", "relations", "evidence", "sessions", "proposed", "decided", @@ -206,6 +212,9 @@ def init(cls, root: Path) -> KBStore: (kb.kb_dir / sub).mkdir(exist_ok=True) if not kb.config_path.exists(): kb.config_path.write_text(_yaml_dump(_starter_config())) + schema_version_file = kb.kb_dir / SCHEMA_VERSION_FILENAME + if not schema_version_file.exists(): + schema_version_file.write_text(SCHEMA_VERSION + "\n") gi = kb.kb_dir / ".gitignore" if not gi.exists(): # state.db is derived; proposed/ is the agent's scratch space. diff --git a/tests/fixtures/migrations/0099-test-rename.yaml b/tests/fixtures/migrations/0099-test-rename.yaml new file mode 100644 index 00000000..0b1091bc --- /dev/null +++ b/tests/fixtures/migrations/0099-test-rename.yaml @@ -0,0 +1,8 @@ +from_version: "0.1.0" +to_version: "0.2.0" +artifact: claims +description: "test fixture: rename a legacy field on claims" +transforms: + - rename: {from: old_note, to: note_tag} +reverse: + - rename: {from: note_tag, to: old_note} diff --git a/tests/test_schema_migrations.py b/tests/test_schema_migrations.py new file mode 100644 index 00000000..b4985a3c --- /dev/null +++ b/tests/test_schema_migrations.py @@ -0,0 +1,250 @@ +"""Tests for the semver model-schema migration runner. + +Covers every acceptance criterion in #200: forward apply + model-validate, +deterministic plan, byte-equivalent rollback, crash-leaves-prior-version, +pending-proposal precondition, and the named synthetic 0099 manifest. The legacy +integer-format `vouch migrate` is exercised separately in test_migrations.py and +must keep passing (the command is now a group with the legacy behavior on the +no-subcommand path). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from click.testing import CliRunner + +from vouch import audit +from vouch import migrations as mig +from vouch.cli import cli +from vouch.migrations import runner, schema +from vouch.migrations.manifest import ManifestError, parse_manifest +from vouch.proposals import propose_claim +from vouch.storage import KBStore, _yaml_dump, _yaml_load + +FIXTURES = Path(__file__).parent / "fixtures" / "migrations" + + +@pytest.fixture +def store(tmp_path, monkeypatch) -> KBStore: + s = KBStore.init(tmp_path) + monkeypatch.chdir(s.root) + return s + + +def _write_claim_yaml(store: KBStore, claim_id: str, data: dict) -> Path: + """Write a raw claim file (lets us inject legacy/extra fields off-model).""" + path = store.kb_dir / "claims" / f"{claim_id}.yaml" + path.write_text(_yaml_dump({"id": claim_id, **data})) + return path + + +def _make_manifest(path: Path, **fields) -> None: + path.write_text(_yaml_dump(fields)) + + +# --- init writes the schema stamp ----------------------------------------- + + +def test_init_writes_schema_version(store: KBStore) -> None: + assert (store.kb_dir / "schema_version").read_text().strip() == "0.1.0" + assert schema.read_schema_version(store) == "0.1.0" + + +def test_absent_schema_version_is_baseline(store: KBStore) -> None: + (store.kb_dir / "schema_version").unlink() + assert schema.read_schema_version(store) == "0.1.0" + + +# --- acceptance #6: the named synthetic 0099 manifest --------------------- + + +def test_0099_rename_hits_right_files_and_rolls_back(store: KBStore) -> None: + touched = _write_claim_yaml(store, "c-legacy", {"text": "t", "old_note": "keep-me"}) + untouched = _write_claim_yaml(store, "c-plain", {"text": "t", "evidence": ["x"]}) + before_touched = touched.read_text() + before_untouched = untouched.read_text() + + plan = runner.plan(store, manifests_dir=FIXTURES) + assert plan["needed"] is True + assert plan["steps"][0]["changed"] == ["claims/c-legacy.yaml"] # only the legacy one + + result = runner.apply(store, manifests_dir=FIXTURES, actor="tester") + assert result["applied"] is True + assert schema.read_schema_version(store) == "0.2.0" + migrated = _yaml_load(touched.read_text()) + assert "old_note" not in migrated and migrated["note_tag"] == "keep-me" + assert untouched.read_text() == before_untouched # untouched stays byte-identical + + runner.rollback(store, actor="tester") + assert schema.read_schema_version(store) == "0.1.0" + assert touched.read_text() == before_touched # byte-equivalent restore + + +# --- acceptance #1: old KB loads cleanly after a multi-step migrate ------- + + +def _chain_manifests(dirpath: Path) -> Path: + dirpath.mkdir(parents=True, exist_ok=True) + _make_manifest( + dirpath / "0001-body-to-text.yaml", + from_version="0.1.0", to_version="0.2.0", artifact="claims", + description="rename body -> text (the required field)", + transforms=[{"rename": {"from": "body", "to": "text"}}], + ) + _make_manifest( + dirpath / "0002-default-confidence.yaml", + from_version="0.2.0", to_version="0.3.0", artifact="claims", + description="default confidence", + transforms=[{"default": {"field": "confidence", "value": 0.7}}], + ) + return dirpath + + +def test_legacy_kb_loads_cleanly_after_migrate(store: KBStore, tmp_path) -> None: + manifests = _chain_manifests(tmp_path / "chain") + # An "old-form" claim: carries `body` instead of the required `text` field, + # so it fails to model-validate until migrated. + _write_claim_yaml(store, "c1", {"body": "old fact", "evidence": ["src-1"]}) + before = runner.verify(store) + assert before["ok"] is False # missing required `text` + + result = runner.apply(store, to_version="0.3.0", manifests_dir=manifests, actor="t") + assert result["to_version"] == "0.3.0" + assert result["manifests"] == ["0001-body-to-text", "0002-default-confidence"] + + after = runner.verify(store) + assert after["ok"] is True, after["errors"] + assert schema.read_schema_version(store) == "0.3.0" + + +# --- acceptance #2: plan is deterministic + byte-identical on repeat ------ + + +def test_plan_is_deterministic(store: KBStore) -> None: + for i in range(5): + _write_claim_yaml(store, f"c{i}", {"text": "t", "old_note": str(i)}) + first = json.dumps(runner.plan(store, manifests_dir=FIXTURES), sort_keys=True) + second = json.dumps(runner.plan(store, manifests_dir=FIXTURES), sort_keys=True) + assert first == second + + +# --- acceptance #3: apply -> rollback is byte-equivalent ------------------ + + +def test_apply_then_rollback_byte_equivalent(store: KBStore) -> None: + for i in range(3): + _write_claim_yaml(store, f"c{i}", {"text": "t", "old_note": f"n{i}"}) + snapshot = {p.name: p.read_text() for p in (store.kb_dir / "claims").glob("*.yaml")} + + runner.apply(store, manifests_dir=FIXTURES, actor="t") + runner.rollback(store, actor="t") + + restored = {p.name: p.read_text() for p in (store.kb_dir / "claims").glob("*.yaml")} + assert restored == snapshot + + +# --- acceptance #4: interrupt mid-run leaves prior version ---------------- + + +def test_crash_mid_apply_leaves_prior_version(store: KBStore) -> None: + for i in range(3): + _write_claim_yaml(store, f"c{i}", {"text": "t", "old_note": f"n{i}"}) + snapshot = {p.name: p.read_text() for p in (store.kb_dir / "claims").glob("*.yaml")} + + with pytest.raises(runner.CrashSimulated): + runner.apply(store, manifests_dir=FIXTURES, actor="t", _fail_after=1) + + # version stamp is bumped last, so a crash leaves the KB at the prior version + assert schema.read_schema_version(store) == "0.1.0" + # ...and the journal lets rollback restore the partially-written files exactly + runner.rollback(store, actor="t") + restored = {p.name: p.read_text() for p in (store.kb_dir / "claims").glob("*.yaml")} + assert restored == snapshot + assert schema.read_schema_version(store) == "0.1.0" + + +# --- acceptance #5: pending proposals block apply ------------------------- + + +def test_apply_refuses_with_pending_proposals(store: KBStore) -> None: + src = store.put_source(b"a source") + propose_claim(store, text="pending one", evidence=[src.id], proposed_by="agent") + _write_claim_yaml(store, "c-legacy", {"text": "t", "old_note": "x"}) + + with pytest.raises(mig.MigrationError) as exc: + runner.apply(store, manifests_dir=FIXTURES, actor="t") + assert "list-pending" in str(exc.value) + + +# --- audit + status ------------------------------------------------------- + + +def test_apply_audits_and_status_reports(store: KBStore) -> None: + _write_claim_yaml(store, "c-legacy", {"text": "t", "old_note": "x"}) + runner.apply(store, manifests_dir=FIXTURES, actor="tester") + events = [e for e in audit.read_events(store.kb_dir) if e.event == "kb.migrate.apply"] + assert events and events[-1].actor == "tester" + + st = runner.status(store, manifests_dir=FIXTURES) + assert st["schema_version"] == "0.2.0" + assert st["up_to_date"] is True + + +# --- manifest validation -------------------------------------------------- + + +def test_manifest_rejects_bad_verb(tmp_path) -> None: + p = tmp_path / "0001-bad.yaml" + _make_manifest(p, from_version="0.1.0", to_version="0.2.0", artifact="claims", + transforms=[{"frobnicate": {"x": 1}}]) + with pytest.raises(ManifestError): + parse_manifest(p) + + +def test_manifest_rejects_backwards_version(tmp_path) -> None: + p = tmp_path / "0001-back.yaml" + _make_manifest(p, from_version="0.3.0", to_version="0.2.0", artifact="claims", + transforms=[]) + with pytest.raises(ManifestError): + parse_manifest(p) + + +# --- CLI ------------------------------------------------------------------ + + +def test_cli_migrate_status_up_to_date(store: KBStore) -> None: + res = CliRunner().invoke(cli, ["migrate", "status", "--json"]) + assert res.exit_code == 0, res.output + data = json.loads(res.output) + assert data["schema_version"] == "0.1.0" and data["up_to_date"] is True + + +def test_cli_migrate_apply_and_verify(store: KBStore, monkeypatch) -> None: + monkeypatch.setenv("VOUCH_MIGRATIONS_DIR", str(FIXTURES)) + _write_claim_yaml(store, "c-legacy", {"text": "t", "old_note": "x"}) + + res = CliRunner().invoke(cli, ["migrate", "apply", "--yes", "--json"]) + assert res.exit_code == 0, res.output + assert json.loads(res.output)["to_version"] == "0.2.0" + + res2 = CliRunner().invoke(cli, ["migrate", "verify", "--json"]) + assert res2.exit_code == 0, res2.output + assert json.loads(res2.output)["ok"] is True + + +def test_cli_migrate_plan_lists_files(store: KBStore, monkeypatch) -> None: + monkeypatch.setenv("VOUCH_MIGRATIONS_DIR", str(FIXTURES)) + _write_claim_yaml(store, "c-legacy", {"text": "t", "old_note": "x"}) + res = CliRunner().invoke(cli, ["migrate", "plan"]) + assert res.exit_code == 0, res.output + assert "claims/c-legacy.yaml" in res.output + + +def test_cli_legacy_migrate_still_works(store: KBStore) -> None: + # `vouch migrate` with no subcommand keeps the legacy integer-format behavior. + res = CliRunner().invoke(cli, ["migrate", "--check"]) + assert res.exit_code == 0, res.output + assert "KB format" in res.output