From fae4bef5e69618909500e6ee11a72c7e74c5c5fb Mon Sep 17 00:00:00 2001 From: josie Date: Wed, 14 Oct 2026 06:48:35 -0400 Subject: [PATCH 1/3] WIP: canonical payload comparison for the selected final stage Co-Authored-By: Claude Opus 5.5 --- .../compare_payloads.py | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 research/selected-final-stage-portability/compare_payloads.py diff --git a/research/selected-final-stage-portability/compare_payloads.py b/research/selected-final-stage-portability/compare_payloads.py new file mode 100644 index 0000000000..88f6a3ce90 --- /dev/null +++ b/research/selected-final-stage-portability/compare_payloads.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Compare two replay output trees file by file, by bytes and by canonical payload. + +Classes: ``bytes`` (identical files), ``gzip-container`` (different .gz bytes, +identical decompressed bytes), ``json-format`` (different bytes, equal parsed +JSON), ``json-diff`` (parsed JSON differs; the differing JSON paths are listed), +``text-diff`` (other files whose bytes differ; differing line numbers listed), +``json-relocated`` (parsed JSON equal once each tree's own absolute input/work +directories are replaced by the same placeholder, in keys and string values), +``missing`` / ``extra``. A remaining JSON difference is reported as ``volatile`` +only when every differing leaf is under a key named in VOLATILE_KEYS (wall-clock +seconds and the digests of logs that print them); everything else is reported +as ``proof-relevant-candidate`` for a human to inspect. + +Standard library only; Python 3.10+. Prepared with Claude Code (Claude Opus 5.5) +for JosieElliston's donated session 26. Apache-2.0. +""" +import argparse +import gzip +import hashlib +import json +from pathlib import Path + +VOLATILE_KEYS = frozenset({'seconds', 'log_sha256'}) + + +def files(root): + return {str(p.relative_to(root)): p for p in sorted(root.rglob('*')) + if p.is_file() and '__pycache__' not in p.parts} + + +def json_diff(a, b, path=''): + if type(a) is not type(b): + return [(path, a, b)] + if isinstance(a, dict): + out = [] + for k in sorted(set(a) | set(b), key=str): + if k not in a or k not in b: + out.append((f'{path}/{k}', a.get(k, ''), b.get(k, ''))) + else: + out += json_diff(a[k], b[k], f'{path}/{k}') + return out + if isinstance(a, list): + if len(a) != len(b): + return [(path + '#len', len(a), len(b))] + out = [] + for i, (x, y) in enumerate(zip(a, b)): + out += json_diff(x, y, f'{path}/{i}') + return out + return [] if a == b else [(path, a, b)] + + +def volatile(diff): + return diff[0].rsplit('/', 1)[-1] in VOLATILE_KEYS + + +def relocate(value, roots): + """Replace absolute directory prefixes (longest first) by placeholders.""" + if isinstance(value, str): + for token, prefix in roots: + if value == prefix or value.startswith(prefix + '/'): + return token + value[len(prefix):] + return value + if isinstance(value, dict): + return {relocate(k, roots): relocate(v, roots) for k, v in value.items()} + if isinstance(value, list): + return [relocate(v, roots) for v in value] + return value + + +def payload(path): + raw = path.read_bytes() + if path.name.endswith('.gz'): + raw = gzip.decompress(raw) + return raw + + +def classify(ref, run, ref_roots, run_roots): + a, b = ref.read_bytes(), run.read_bytes() + if a == b: + return dict(cls='bytes') + pa, pb = payload(ref), payload(run) + gz = ref.name.endswith('.gz') + if gz and pa == pb: + return dict(cls='gzip-container') + name = ref.name[:-3] if gz else ref.name + if name.endswith('.json'): + ja, jb = json.loads(pa), json.loads(pb) + if ja == jb: + return dict(cls='json-format', gzip=gz) + ja, jb = relocate(ja, ref_roots), relocate(jb, run_roots) + if ja == jb: + return dict(cls='json-relocated', gzip=gz) + diffs = json_diff(ja, jb) + rel = [d for d in diffs if not volatile(d)] + return dict(cls='json-diff', gzip=gz, differing=len(diffs), + verdict='volatile' if not rel else 'proof-relevant-candidate', + examples=[[d[0], repr(d[1])[:120], repr(d[2])[:120]] for d in (rel or diffs)[:8]]) + la, lb = pa.decode(errors='replace').splitlines(), pb.decode(errors='replace').splitlines() + lines = [i + 1 for i in range(max(len(la), len(lb))) + if (la[i] if i < len(la) else None) != (lb[i] if i < len(lb) else None)] + return dict(cls='text-diff', lines=len(la), run_lines=len(lb), differing_lines=lines[:20], + examples=[[(la[i-1] if i <= len(la) else '')[:160], + (lb[i-1] if i <= len(lb) else '')[:160]] for i in lines[:4]]) + + +def roots(pairs): + out = [] + for pair in pairs: + token, _, prefix = pair.partition('=') + if not token or not prefix.startswith('/'): + raise SystemExit('expected TOKEN=/absolute/prefix: ' + pair) + out.append((token, prefix.rstrip('/'))) + return sorted(out, key=lambda t: -len(t[1])) + + +def compare(ref_root, run_root, ref_roots=(), run_roots=()): + ref, run = files(ref_root), files(run_root) + out = {} + for name in sorted(set(ref) | set(run)): + if name not in run: + out[name] = dict(cls='missing') + elif name not in ref: + out[name] = dict(cls='extra') + else: + out[name] = classify(ref[name], run[name], ref_roots, run_roots) + out[name]['ref_sha256'] = hashlib.sha256(ref[name].read_bytes()).hexdigest() + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('ref', type=Path) + ap.add_argument('run', type=Path) + ap.add_argument('--ref-root', action='append', default=[], metavar='TOKEN=PREFIX', + help='absolute directory used by the reference run, mapped to TOKEN') + ap.add_argument('--run-root', action='append', default=[], metavar='TOKEN=PREFIX', + help='absolute directory used by the compared run, mapped to TOKEN') + ap.add_argument('--out', type=Path) + a = ap.parse_args() + result = compare(a.ref, a.run, roots(a.ref_root), roots(a.run_root)) + counts = {} + for v in result.values(): + k = v['cls'] + (':' + v['verdict'] if 'verdict' in v else '') + counts[k] = counts.get(k, 0) + 1 + report = dict(ref=str(a.ref), run=str(a.run), counts=counts, files=result) + if a.out: + a.out.write_text(json.dumps(report, indent=1, sort_keys=True) + '\n') + print(json.dumps(counts, sort_keys=True)) + + +if __name__ == '__main__': + main() From 0b0835380e093fa8dc3f310a2457563baa516515 Mon Sep 17 00:00:00 2001 From: josie Date: Wed, 14 Oct 2026 06:52:40 -0400 Subject: [PATCH 2/3] Research: interpreter matrix and canonical-payload comparison for the selected final stage (no new kappa) Reruns research/complex-t16-rare-pattern-exchanges/verify.py from the hash-bound archived #1042 slots/local and compares every output with the reviewed #1094 archive: 102/105 byte-identical, one relocated path map, two files differing only in wall-clock seconds, on CPython 3.10-3.14. Co-Authored-By: Claude Opus 5.5 --- .../README.md | 146 ++++++++++++ .../SOURCES.json | 66 ++++++ .../TASK.json | 72 ++++++ .../VERIFY.json | 27 +++ .../verify.py | 209 ++++++++++++++++++ 5 files changed, 520 insertions(+) create mode 100644 research/selected-final-stage-portability/README.md create mode 100644 research/selected-final-stage-portability/SOURCES.json create mode 100644 research/selected-final-stage-portability/TASK.json create mode 100644 research/selected-final-stage-portability/VERIFY.json create mode 100644 research/selected-final-stage-portability/verify.py diff --git a/research/selected-final-stage-portability/README.md b/research/selected-final-stage-portability/README.md new file mode 100644 index 0000000000..a2449763db --- /dev/null +++ b/research/selected-final-stage-portability/README.md @@ -0,0 +1,146 @@ +# Selected final stage: interpreter matrix and canonical-payload comparison + +**Verifier engineering / portability receipt; no new kappa, no change to any +selected, pinned or frozen file.** Card `portable-replay` (priority revision 3, +chooser seed `e95bc9807d1d236e2d6d663b2945e09b`), refined to one stage and one +comparison; see [TASK.json](TASK.json). Base `0b1cccf8`. + +## Question + +The selected #1094 replay (`make frontier1094-verify`) has exactly one stage +after its #1042 baseline: +[`complex-t16-rare-pattern-exchanges/verify.py`](../complex-t16-rare-pattern-exchanges/verify.py), +which runs the frozen PR1087 strict verifier on Rohan Arun's owner exchanges. +Its README says "Python 3.12+", and the selected-replay CI pins 3.12. Does the +stage pass on other CPython versions, and is its output the same as the +reviewed run archived in +[`community-frontier1094-audit/selected-replay.tar.gz`](../community-frontier1094-audit/selected-replay.tar.gz)? +If the output differs, is the difference in proof-relevant content, or only in +container bytes, absolute paths and timings? + +## Result + +The stage was rerun from the committed #1042 `slots/local` (all ten files match +the #1042 receipt's hashes). It passed on all five interpreters and five +hash-seed settings, with kappa `0.001672660197076918` each time. Each rerun's +105 output files compare with the reviewed run as follows: + +| Class | Files | What differs | +|---|---:|---| +| byte-identical | 102 | nothing: all 22 `.gz` files (gzip containers included), 69 of the 72 `.json` files and all 11 inner stage logs | +| `json-relocated` | 1 | `INDEPENDENT-T16.json`: only the absolute paths used as keys of `input_sha256`; all six input digests agree | +| volatile | 2 | `REPLAY.json`, `STAGES.json`: only the 11 per-stage `seconds` values; every `log_sha256` agrees | + +| Interpreter (macOS 26.6.2 arm64, uv-managed CPython) | `PYTHONHASHSEED` | Stage wall time | Classification | +|---|---|---:|---| +| 3.10.21 | random | 19.7 s | 102 / 1 / 2 | +| 3.11.16 | random | 17.5 s | 102 / 1 / 2 | +| 3.12.14 | random, 0, 1, 4242 | 16.6 s | 102 / 1 / 2 (all four) | +| 3.13.15 | random | 17.5 s | 102 / 1 / 2 | +| 3.14.7 | random, 0, 987654321 | 14.6 s | 102 / 1 / 2 (all three) | + +So the final stage is deterministic and portable across CPython 3.10–3.14 on +this platform, down to compressed bytes. "Python 3.12+" in the stage README is +conservative. The real floor is `tarfile.extractall(filter='data')`, which +needs 3.10.12+, 3.11.4+ or 3.12+. + +**One receipt field cannot be reproduced.** The outer stage log +`rare-pattern-exchanges.log` prints per-stage wall-clock seconds, so the #1094 +receipt's `log_sha256` for that stage (`80362b2c…`) cannot match any rerun. The +cold replay never compares against it, so this does not break replay. A reviewer +comparing two #1094 receipts should still expect that field to differ. The inner +`log_sha256` values in `candidate/strict/STAGES.json` do not contain timings, and +they are reproduced exactly. The verifier here compares the outer log only after +removing the trailing seconds field from each `PASS ` line. + +## What the verifier checks + +[`verify.py`](verify.py) runs in about 20 s using only the standard library: + +1. It extracts `slots/local` from the #1042 archive and requires exactly the ten + files bound in the #1042 `RESULT.json`, with matching hashes. +2. It extracts the reviewed output from the #1094 archive. It binds that output + to the committed #1094 `RESULT.json`, `STAGES.json`, `REPLAY.json` and + `assembly_sha256`. It also checks that the #1094 receipt names the committed + #1042 receipt and the stage script's current hash. +3. It reruns the stage with the active interpreter in a fresh directory, then + compares the result with [`compare_payloads.py`](compare_payloads.py), which + works file by file, by bytes and by canonical payload. It requires the exact + classification above. +4. It runs negative controls on copies of the rerun: + - Each of these must be flagged `proof-relevant-candidate`: + - a changed proof value (`live` in `INDEPENDENT-T16.json`); + - a changed kappa in `REPLAY.json`, which also differs in volatile seconds; + - a changed gzip payload; + - an input path outside the mapped roots. + - A missing file must be reported `missing`. + - A gzip file re-compressed at a different level and mtime must be accepted + as `gzip-container`. + +`compare_payloads.py` can be reused on its own: + +```sh +python3 research/selected-final-stage-portability/compare_payloads.py REF_DIR RUN_DIR \ + --ref-root '=/abs/ref/work' --run-root '=/abs/run/work' --out report.json +``` + +A JSON difference counts as `volatile` only when every differing leaf sits under +a key in `VOLATILE_KEYS` (`seconds`, `log_sha256`). Every other difference is +reported as `proof-relevant-candidate` for a person to inspect. It does not +decide which fields are proof-relevant; it lists the paths that differ. + +## Reproduce + +```sh +python3 scripts/research_package.py --package research/selected-final-stage-portability --check-only +python3.12 scripts/research_package.py --package research/selected-final-stage-portability +# matrix: +for v in 3.10 3.11 3.12 3.13 3.14; do python$v -B research/selected-final-stage-portability/verify.py; done +``` + +## Not checked + +- **Earlier stages.** The cold #1042 and #799 baselines were not rebuilt; the + stage input is the hash-bound archived `slots/local`. The 711 s `chain` stage + and the native steps are outside this scope. +- **Other platforms.** Linux, Windows and non-uv builds of CPython were not + run. The reviewed run's `/private/tmp` paths suggest it ran on macOS too, so + this is not a cross-OS result. +- **Transitive inputs.** The stage reads files from `--main` (the repository) + that are bound by the PR1087 verifier's own pins. `VERIFY.json` declares the + stage script, its manifest and the receipts/archives, not every transitive + file. +- **Mathematics.** This does not check the mathematics of the stage or of its + inherited assumptions, and it does not replace the selected cold replay. + +## Open next questions + +1. **Does the same byte-level agreement hold on Linux (CI's Ubuntu 3.12) and on + Windows?** *Why it matters:* if it holds, a cross-OS reviewer could compare + receipts directly instead of rerunning the stage. *First step:* run + `verify.py` on Ubuntu 3.12. *Blocked by:* no Linux host in this session. +2. **Can the #1094 receipt drop or annotate its non-reproducible outer + `log_sha256`?** *Why it matters:* it would make two runs' receipts directly + comparable. *First step:* a maintainer decision. This package changes no + selected script. *Blocked by:* any edit to `scripts/verify_frontier1094.py` + touches the selected reproduction path. +3. **Do the same classes hold for the #1042 stages?** *Why it matters:* the + #1042 receipt binds 457 artifact hashes, so byte determinism there would let + reuse mode be checked across machines. *First step:* run + `compare_payloads.py` on a cold `verify_frontier1042.py` run against + `community-frontier1042-audit/selected-replay.tar.gz`. *Blocked by:* a cold + run takes about 20 minutes plus the #799 baseline. + +## Credit + +The stage under test is Rohan Arun's #1094 (`complex-t16-rare-pattern-exchanges`) +on Jeremy Kothe's PR1087 strict verifier and the credited PR1018/1005/869/799/741 +lineage. Douglas Colkitt wrote the selected replay driver and the receipts that +the reviewed archives come from. The gzip-container convention follows Maxime +Fleury's [reproducibility audit](../gzip-container-reproducibility/README.md), +which built on Antoine Olivier's replay findings. Related audits, which this +package does not repeat: [ancestry binding, seed 69](../frontier1094-ancestry-binding-seed69/README.md) +and PR1735 (certificate-to-replay binding). + +Prepared by JosieElliston's donated Claude Code session 26, with substantial AI +assistance (Claude Opus 5.5), self-reviewed only. Apache-2.0. diff --git a/research/selected-final-stage-portability/SOURCES.json b/research/selected-final-stage-portability/SOURCES.json new file mode 100644 index 0000000000..8a6658d3e4 --- /dev/null +++ b/research/selected-final-stage-portability/SOURCES.json @@ -0,0 +1,66 @@ +{ + "schema_version": 1, + "base_commit": "0b1cccf86e2f5132ed286a26ef0a681a495740ea", + "files": [ + { + "path": "certificates/selected-result.json", + "usage": "provenance", + "git_blob": "ae6a99f28aa468649eb66a361e764d769363bf11", + "sha256": "10b9e340fd8e354a37a0c4813c90e5de1dd42044b5f6aca4a3fc5a42b685fa32" + }, + { + "path": "docs/research/agent-task-catalog.json", + "usage": "provenance", + "git_blob": "06c96f5579bb67ed1b86d334e78fbed7da6945f1", + "sha256": "6686ad58f1fbdbc87134a78901b60fd53a20d3984355e91defbbebc9e9fa8075" + }, + { + "path": "research/community-frontier1042-audit/RESULT.json", + "usage": "input", + "git_blob": "8182d0818c0cbda7e52c10120f15b2c3a2c8f933", + "sha256": "bf12c6ed3b412539638b76addb40591a631609b2ab9e5ee0db1f6bbe067d0ec0" + }, + { + "path": "research/community-frontier1042-audit/selected-replay.tar.gz", + "usage": "input", + "git_blob": "6f4dc6fe7631d8aacf8081fcaecf32197181685b", + "sha256": "22c3f4dce4b897a1d423d1388801933e288b6a6b1906cb15e1b16a5eb68db754" + }, + { + "path": "research/community-frontier1094-audit/REPLAY.json", + "usage": "input", + "git_blob": "2cfb7fffa7cd78f418be16c13224c3206b6d0183", + "sha256": "ce0678fa7abaab221de74baeace0c6696d5ea23a36719a58cff230215a06aae7" + }, + { + "path": "research/community-frontier1094-audit/RESULT.json", + "usage": "input", + "git_blob": "131d9c96a54632ae385ced4138fd7f30ad448a3e", + "sha256": "1e3e47e9ec24bcb6fb5f703b7fce0aab658d84ff72bfae5da3ae429d17b6e752" + }, + { + "path": "research/community-frontier1094-audit/STAGES.json", + "usage": "input", + "git_blob": "a92499421609e4cdb5630f2697d2bad4b5a42a83", + "sha256": "72ad3ba266488da6bb6a41be404700dcea183a85002a5428df0033685c56d8be" + }, + { + "path": "research/community-frontier1094-audit/selected-replay.tar.gz", + "usage": "input", + "git_blob": "c24daf067f9c543421d89ca2eeaa4eb78a57fb6f", + "sha256": "f1da80610bf33c58527f18ad63b4350d0b864e1d4486141ee17ac9bed4d529d6" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/MANIFEST.json", + "usage": "input", + "git_blob": "3216666bd231516f8d2387e3096d6a5bd413b519", + "sha256": "16a964621e718c8da4c859ccbb280c3c00816ca72874e955910fdf0ef27a0adc" + }, + { + "path": "research/complex-t16-rare-pattern-exchanges/verify.py", + "usage": "input", + "git_blob": "9d471cf90835bc91d32bb0216c2bb867011a7420", + "sha256": "b8c2f76b305c5c9ceb42df203deef564857d17f1f8182d0745dda9ab96e3d4de" + } + ] +} diff --git a/research/selected-final-stage-portability/TASK.json b/research/selected-final-stage-portability/TASK.json new file mode 100644 index 0000000000..72d2ec53f1 --- /dev/null +++ b/research/selected-final-stage-portability/TASK.json @@ -0,0 +1,72 @@ +{ + "schema_version": 1, + "task_id": "portable-replay", + "priority_revision": 3, + "base_commit": "0b1cccf86e2f5132ed286a26ef0a681a495740ea", + "original_card": { + "id": "portable-replay", + "question": "Can one reproducible environment or artifact-comparison failure be repaired?", + "starting_points": [ + "research/gzip-container-reproducibility/README.md", + "docs/ci-verification.md" + ], + "methods": [ + "Reproduce one portability failure and add a minimal cross-environment fix.", + "Audit one canonical-payload comparison for omitted proof-relevant fields." + ], + "acceptance_gate": "Preserve immutable source hashes and proof-relevant content; a different compressed byte stream alone is not a mathematical mismatch.", + "first_step": { + "command": [ + "python3", + "-B", + "scripts/research_baseline.py" + ], + "expected": "PASS_RESEARCH_BASELINE_BINDINGS_ONLY", + "inspect": "Read the gzip-container reproducibility examples and distinguish frozen source bytes from regenerated payloads.", + "scope": "Positive control / source-bound orientation only; this is not a new construction or full cold replay." + }, + "first_experiment": "Reproduce one specific mismatch or audit one canonical comparison for omitted proof-relevant fields.", + "bottleneck_link": "Portable exact checks prevent environment failures from blocking valid contributions.", + "allowed_variations": [ + "One interpreter/version mismatch or one payload field comparison." + ], + "scope_change_triggers": [ + "Removing immutable-source pins because generated compressed bytes differ." + ], + "overlap_terms": [ + "gzip container", + "reproducibility", + "canonical payload" + ] + }, + "scope_status": "refined", + "actual_scope": "Interpreter matrix (CPython 3.10-3.14, five PYTHONHASHSEED settings, macOS arm64) for the selected #1094 record's only post-#1042 stage, research/complex-t16-rare-pattern-exchanges/verify.py, run from the hash-bound archived #1042 slots/local, with a file-by-file byte and canonical-payload comparison against the reviewed run archived in research/community-frontier1094-audit/selected-replay.tar.gz.", + "scope_change_reason": "The card asks for one portability failure or one canonical-payload comparison; this picks the selected record's final stage as the object and checks whether any interpreter or hash seed changes its output payload, and which receipt fields are inherently non-reproducible. No failure was found to repair, so the result is a portability receipt plus a reusable comparison tool rather than a fix.", + "bottleneck_link": "Side result for review throughput: shows a reviewer can rerun the selected final stage on any CPython 3.10-3.14 and expect byte-identical artifacts apart from one relocated path map and timings, and identifies the one #1094 receipt hash (outer stage log) that no rerun can reproduce.", + "budget_minutes": 60, + "stop_rule": "One stage, one platform, five interpreters; no cold #1042/#799 rebuild and no edit to selected or frozen scripts.", + "outcome": "completed", + "evidence": "All 5 interpreters and 5 hash-seed runs pass with kappa 0.001672660197076918; of 105 outputs, 102 are byte-identical to the reviewed run (all 22 .gz, all 11 inner logs), INDEPENDENT-T16.json is equal after relocating absolute path keys, REPLAY.json/STAGES.json differ only in 11 'seconds' values. verify.py reruns the stage (~20 s) and checks this classification plus six negative controls.", + "author": "JosieElliston (donated Claude Code session 26, Claude Opus 5.5)", + "ai_assistance": "Claude Code (Claude Opus 5.5) wrote the scripts and text in JosieElliston's donated session 26; self-reviewed only, no independent AI or human review.", + "open_questions": [ + { + "question": "Does the same byte-level agreement hold on Linux (Ubuntu CI, Python 3.12) and Windows?", + "why_it_matters": "Then a cross-OS reviewer could compare receipts directly instead of rerunning.", + "first_step": "Run verify.py on Ubuntu with Python 3.12.", + "blocked_by": "No Linux host in this session." + }, + { + "question": "Should the #1094 receipt drop or annotate its non-reproducible outer log_sha256 (the stage log prints wall-clock seconds)?", + "why_it_matters": "Makes two runs' receipts directly comparable.", + "first_step": "Maintainer decision; this package changes no selected script.", + "blocked_by": "Any edit to scripts/verify_frontier1094.py touches the selected reproduction path." + }, + { + "question": "Do the #1042 stages' 457 bound artifacts reproduce byte for byte across interpreters?", + "why_it_matters": "Would let reuse mode be checked against the published receipt on another machine.", + "first_step": "Run compare_payloads.py on a cold verify_frontier1042.py run against community-frontier1042-audit/selected-replay.tar.gz.", + "blocked_by": "A cold run takes about 20 minutes plus the #799 baseline." + } + ] +} diff --git a/research/selected-final-stage-portability/VERIFY.json b/research/selected-final-stage-portability/VERIFY.json new file mode 100644 index 0000000000..621c8059b0 --- /dev/null +++ b/research/selected-final-stage-portability/VERIFY.json @@ -0,0 +1,27 @@ +{ + "schema_version": 1, + "status": "ready", + "python": "3.12", + "requirements": [], + "inputs": [ + "research/community-frontier1042-audit/RESULT.json", + "research/community-frontier1042-audit/selected-replay.tar.gz", + "research/community-frontier1094-audit/REPLAY.json", + "research/community-frontier1094-audit/RESULT.json", + "research/community-frontier1094-audit/STAGES.json", + "research/community-frontier1094-audit/selected-replay.tar.gz", + "research/complex-t16-rare-pattern-exchanges/MANIFEST.json", + "research/complex-t16-rare-pattern-exchanges/verify.py" + ], + "commands": [ + { + "argv": [ + "{python}", + "-B", + "{package}/verify.py" + ], + "timeout_seconds": 300, + "checks": "Binds slots/local to the #1042 receipt and the archived reviewed output to the #1094 receipts; reruns the final stage; requires 102 byte-identical, 1 json-relocated and 2 seconds-only files; six negative controls (proof value, kappa, gzip payload, unmapped path flagged; missing file reported; re-compressed gzip accepted)." + } + ] +} diff --git a/research/selected-final-stage-portability/verify.py b/research/selected-final-stage-portability/verify.py new file mode 100644 index 0000000000..2ec1a54a70 --- /dev/null +++ b/research/selected-final-stage-portability/verify.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Rerun the selected record's final stage and compare it with the reviewed run. + +The selected #1094 replay has one stage after the #1042 baseline: +``research/complex-t16-rare-pattern-exchanges/verify.py`` (the frozen PR1087 +strict verifier on Rohan Arun's owner exchanges). This script + +1. extracts its ``--local`` input (``slots/local``) from the committed #1042 + replay archive and checks all ten files against the #1042 receipt; +2. extracts the reviewed #1094 output from the committed #1094 archive and binds + it to the committed #1094 receipts; +3. reruns the stage with the active interpreter in a fresh directory; +4. compares every output file by bytes and canonical payload + (``compare_payloads.py``) and requires the recorded classification: + 102 byte-identical files, ``INDEPENDENT-T16.json`` equal after relocating + absolute paths, and ``REPLAY.json``/``STAGES.json`` differing only in + wall-clock ``seconds``; +5. runs negative controls showing that the comparison flags a changed + proof-relevant value, a changed gzip payload, a missing file and an + unmapped path, and accepts a re-compressed gzip container. + +Standard library only; Python 3.10+ (``tarfile`` ``filter=`` needs 3.10.12+ or +3.11.4+). Prepared with Claude Code (Claude Opus 5.5) for JosieElliston's +donated session 26. Apache-2.0. +""" +import argparse +import gzip +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tarfile +import tempfile + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[1] +sys.path.insert(0, str(HERE)) +from compare_payloads import compare, roots # noqa: E402 + +A1042 = ROOT / 'research/community-frontier1042-audit' +A1094 = ROOT / 'research/community-frontier1094-audit' +STAGE = ROOT / 'research/complex-t16-rare-pattern-exchanges/verify.py' +# Absolute directories used by the reviewed run, as recorded in its receipts. +REF_WORK = '/private/tmp/batch1099-selected/candidate' +REF_LOCAL = '/private/tmp/batch1047-selected/slots/local' +KAPPA = '0.001672660197076918' +EXPECTED = {'bytes': 102, 'json-relocated': 1, 'json-diff:volatile': 2} +NON_BYTE = {'INDEPENDENT-T16.json': 'json-relocated', 'REPLAY.json': 'json-diff', + 'STAGES.json': 'json-diff'} + + +def sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def check(cond, msg): + if not cond: + raise SystemExit('FAIL: ' + msg) + + +def extract(archive, dest, wanted=None): + with tarfile.open(archive) as t: + members = [m for m in t.getmembers() if wanted is None or m.name.startswith(wanted)] + for m in members: + check(m.isfile() or m.isdir(), 'unexpected archive member ' + m.name) + check((dest / m.name).resolve().is_relative_to(dest.resolve()), 'unsafe member ' + m.name) + t.extractall(dest, members=members, filter='data') + + +def counts(result): + out = {} + for v in result.values(): + k = v['cls'] + (':' + v['verdict'] if 'verdict' in v else '') + out[k] = out.get(k, 0) + 1 + return out + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('--work', type=Path, help='fresh directory (default: a temporary one)') + ap.add_argument('--keep', action='store_true', help='keep the temporary directory') + a = ap.parse_args() + check(not sys.flags.optimize, 'assertions are required by the rerun stage') + if a.work: + work = a.work.resolve() + check(not work.exists(), 'use a fresh --work directory') + work.mkdir(parents=True) + else: + work = Path(tempfile.mkdtemp(prefix='final-stage-portability.')).resolve() + try: + run(work) + finally: + if not a.work and not a.keep: + shutil.rmtree(work, ignore_errors=True) + + +def run(work): + # 1. Input bound to the #1042 receipt. + receipt1042 = json.loads((A1042 / 'RESULT.json').read_bytes()) + inp = work / 'in1042' + inp.mkdir() + extract(A1042 / 'selected-replay.tar.gz', inp, 'slots/local/') + local = inp / 'slots/local' + bound = {k: v for k, v in receipt1042['artifacts'].items() if k.startswith('slots/local/')} + present = {'slots/local/' + p.name for p in local.iterdir()} + check(len(bound) == 10 and set(bound) == present, 'slots/local inventory differs from the #1042 receipt') + for name, digest in bound.items(): + check(sha(inp / name) == digest, '#1042 artifact hash: ' + name) + # 2. Reference output bound to the committed #1094 receipts. + ref = work / 'ref1094' + ref.mkdir() + extract(A1094 / 'selected-replay.tar.gz', ref) + for name in ('RESULT.json', 'STAGES.json'): + check((ref / name).read_bytes() == (A1094 / name).read_bytes(), 'archived #1094 ' + name) + check((ref / 'candidate/strict/REPLAY.json').read_bytes() == (A1094 / 'REPLAY.json').read_bytes(), + 'archived #1094 REPLAY.json') + result1094 = json.loads((A1094 / 'RESULT.json').read_bytes()) + check(sha(ref / 'candidate/strict/ASSEMBLY.json') == result1094['assembly_sha256'], 'archived #1094 assembly') + check(sha(A1042 / 'RESULT.json') == result1094['baseline_receipt_sha256'], '#1094 names the #1042 receipt') + check(result1094['baseline'] + '/slots/local' == REF_LOCAL, 'reviewed --local path') + stage = [s for s in result1094['stages'] if s['name'] == 'rare-pattern-exchanges'] + check(len(stage) == 1 and stage[0]['script_sha256'] == sha(STAGE), 'final-stage script is the reviewed one') + # 3. Rerun with the active interpreter. + out = work / 'run' + log = work / 'rare-pattern-exchanges.log' + env = dict(os.environ, PYTHONDONTWRITEBYTECODE='1') + env.pop('PYTHONOPTIMIZE', None) + with log.open('w') as stream: + p = subprocess.run([sys.executable, '-B', str(STAGE), '--main', str(ROOT), '--local', str(local), + '--work', str(out)], cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT) + text = log.read_text() + check(p.returncode == 0 and text.endswith('PASS_T16_RARE_PATTERN_EXCHANGES ' + KAPPA + '\n'), + 'final stage did not pass; see ' + str(log)) + # The outer stage log prints wall-clock seconds, so its #1094 log_sha256 is + # not reproducible; the inner logs are compared byte for byte below. + ref_log = (ref / 'rare-pattern-exchanges.log').read_text().splitlines() + run_log = text.splitlines() + strip = lambda lines: [l.rsplit(' ', 1)[0] if l.startswith('PASS ') and l.count(' ') == 2 else l for l in lines] + check(strip(ref_log) == strip(run_log), 'outer stage log differs beyond per-stage seconds') + # 4. Canonical comparison. + ref_roots = roots(['=' + REF_WORK, '=' + REF_LOCAL]) + run_roots = roots(['=' + str(out), '=' + str(local)]) + res = compare(ref / 'candidate/strict', out / 'strict', ref_roots, run_roots) + check(counts(res) == EXPECTED, 'classification differs: ' + json.dumps(counts(res), sort_keys=True)) + for name, cls in NON_BYTE.items(): + check(res[name]['cls'] == cls, 'unexpected class for ' + name) + for name in ('REPLAY.json', 'STAGES.json'): + check(res[name]['verdict'] == 'volatile' and res[name]['differing'] == 11, name + ' differs beyond seconds') + print('PASS comparison', json.dumps(counts(res), sort_keys=True), flush=True) + # 5. Negative controls on copies of the rerun output. + controls(ref / 'candidate/strict', out / 'strict', ref_roots, run_roots, work / 'controls') + print('PASS_SELECTED_FINAL_STAGE_PORTABILITY', sys.version.split()[0], KAPPA) + + +def controls(ref, run, ref_roots, run_roots, base): + def variant(name, mutate, expect_file, expect): + d = base / name + shutil.copytree(run, d) + mutate(d) + # File contents still name the rerun's own directories, so the roots are unchanged. + res = compare(ref, d, ref_roots, run_roots) + got = res[expect_file]['cls'] + (':' + res[expect_file]['verdict'] if 'verdict' in res[expect_file] else '') + check(got == expect, f'control {name}: {expect_file} classified {got}, expected {expect}') + print('PASS control', name, '->', got, flush=True) + + def edit_json(rel, fn): + def mutate(d): + p = d / rel + j = json.loads(p.read_bytes()) + fn(j) + p.write_text(json.dumps(j, indent=2) + '\n') + return mutate + + def bump_profile(j): + j['live'] += 1 + variant('proof-value', edit_json('INDEPENDENT-T16.json', bump_profile), + 'INDEPENDENT-T16.json', 'json-diff:proof-relevant-candidate') + + def kappa(j): + j['kappa_decimal'] = j['kappa_decimal'][:-1] + '9' + variant('kappa-with-seconds', edit_json('REPLAY.json', kappa), + 'REPLAY.json', 'json-diff:proof-relevant-candidate') + gz = 'global/hosted/GLOBAL-TEMPLATE.json.gz' + + def recompress(d): + p = d / gz + p.write_bytes(gzip.compress(gzip.decompress(p.read_bytes()), compresslevel=1, mtime=12345)) + variant('gzip-recompressed', recompress, gz, 'gzip-container') + + def gz_payload(d): + p = d / gz + j = json.loads(gzip.decompress(p.read_bytes())) + j = {'tampered': True, **j} if isinstance(j, dict) else j + [0] + p.write_bytes(gzip.compress(json.dumps(j).encode(), mtime=0)) + variant('gzip-payload', gz_payload, gz, 'json-diff:proof-relevant-candidate') + variant('missing-file', lambda d: (d / 'ASSEMBLY.json').unlink(), 'ASSEMBLY.json', 'missing') + + def foreign_path(j): + k = next(iter(j['input_sha256'])) + j['input_sha256']['/elsewhere/' + k.rsplit('/', 1)[-1]] = j['input_sha256'].pop(k) + variant('unmapped-path', edit_json('INDEPENDENT-T16.json', foreign_path), + 'INDEPENDENT-T16.json', 'json-diff:proof-relevant-candidate') + + +if __name__ == '__main__': + main() From ad5b1248f25a71983b20617e2f3a80be22f03990 Mon Sep 17 00:00:00 2001 From: josie Date: Wed, 14 Oct 2026 06:57:08 -0400 Subject: [PATCH 3/3] Bind rerun ASSEMBLY.json to the committed #1094 certificate bytes; inner log-digest control Co-Authored-By: Claude Opus 5.5 --- research/selected-final-stage-portability/README.md | 11 ++++++++++- .../selected-final-stage-portability/SOURCES.json | 6 ++++++ research/selected-final-stage-portability/TASK.json | 2 +- .../selected-final-stage-portability/VERIFY.json | 3 ++- .../compare_payloads.py | 10 +++++++--- research/selected-final-stage-portability/verify.py | 12 ++++++++++-- 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/research/selected-final-stage-portability/README.md b/research/selected-final-stage-portability/README.md index a2449763db..f14313df13 100644 --- a/research/selected-final-stage-portability/README.md +++ b/research/selected-final-stage-portability/README.md @@ -39,6 +39,13 @@ hash-seed settings, with kappa `0.001672660197076918` each time. Each rerun's | 3.13.15 | random | 17.5 s | 102 / 1 / 2 | | 3.14.7 | random, 0, 987654321 | 14.6 s | 102 / 1 / 2 (all three) | +In particular, every rerun's `strict/ASSEMBLY.json` is byte-identical to +`community-frontier1094-audit/ASSEMBLY.json` (`1122cb3a…`). That file is the +certificate the selected record pins at base `0b1cccf8`. PR1735 points out that +`make frontier1094-verify` binds that certificate to the replay only through +kappa. This rerun closes the gap for the final stage, by bytes, on every +interpreter tested. `verify.py` checks it explicitly. + So the final stage is deterministic and portable across CPython 3.10–3.14 on this platform, down to compressed bytes. "Python 3.12+" in the stage README is conservative. The real floor is `tarfile.extractall(filter='data')`, which @@ -71,6 +78,7 @@ removing the trailing seconds field from each `PASS ` line. - Each of these must be flagged `proof-relevant-candidate`: - a changed proof value (`live` in `INDEPENDENT-T16.json`); - a changed kappa in `REPLAY.json`, which also differs in volatile seconds; + - a changed inner `log_sha256` in `STAGES.json`; - a changed gzip payload; - an input path outside the mapped roots. - A missing file must be reported `missing`. @@ -85,7 +93,8 @@ python3 research/selected-final-stage-portability/compare_payloads.py REF_DIR RU ``` A JSON difference counts as `volatile` only when every differing leaf sits under -a key in `VOLATILE_KEYS` (`seconds`, `log_sha256`). Every other difference is +a key in `VOLATILE_KEYS` (only `seconds` by default; `--volatile-key` adds more, +for example `log_sha256` when logs print timings). Every other difference is reported as `proof-relevant-candidate` for a person to inspect. It does not decide which fields are proof-relevant; it lists the paths that differ. diff --git a/research/selected-final-stage-portability/SOURCES.json b/research/selected-final-stage-portability/SOURCES.json index 8a6658d3e4..9dca1ea2de 100644 --- a/research/selected-final-stage-portability/SOURCES.json +++ b/research/selected-final-stage-portability/SOURCES.json @@ -26,6 +26,12 @@ "git_blob": "6f4dc6fe7631d8aacf8081fcaecf32197181685b", "sha256": "22c3f4dce4b897a1d423d1388801933e288b6a6b1906cb15e1b16a5eb68db754" }, + { + "path": "research/community-frontier1094-audit/ASSEMBLY.json", + "usage": "input", + "git_blob": "5fd676df060f3b450782fdf5ff5f6beb63a5eeb0", + "sha256": "1122cb3a575f8c788345fc1bf4c108ac7c3250fb81a5988671f5eb04b3ab3045" + }, { "path": "research/community-frontier1094-audit/REPLAY.json", "usage": "input", diff --git a/research/selected-final-stage-portability/TASK.json b/research/selected-final-stage-portability/TASK.json index 72d2ec53f1..34be033cac 100644 --- a/research/selected-final-stage-portability/TASK.json +++ b/research/selected-final-stage-portability/TASK.json @@ -46,7 +46,7 @@ "budget_minutes": 60, "stop_rule": "One stage, one platform, five interpreters; no cold #1042/#799 rebuild and no edit to selected or frozen scripts.", "outcome": "completed", - "evidence": "All 5 interpreters and 5 hash-seed runs pass with kappa 0.001672660197076918; of 105 outputs, 102 are byte-identical to the reviewed run (all 22 .gz, all 11 inner logs), INDEPENDENT-T16.json is equal after relocating absolute path keys, REPLAY.json/STAGES.json differ only in 11 'seconds' values. verify.py reruns the stage (~20 s) and checks this classification plus six negative controls.", + "evidence": "All 5 interpreters and 5 hash-seed runs pass with kappa 0.001672660197076918; of 105 outputs, 102 are byte-identical to the reviewed run (all 22 .gz, all 11 inner logs), INDEPENDENT-T16.json is equal after relocating absolute path keys, REPLAY.json/STAGES.json differ only in 11 'seconds' values. verify.py reruns the stage (~20 s) and checks this classification plus seven negative controls.", "author": "JosieElliston (donated Claude Code session 26, Claude Opus 5.5)", "ai_assistance": "Claude Code (Claude Opus 5.5) wrote the scripts and text in JosieElliston's donated session 26; self-reviewed only, no independent AI or human review.", "open_questions": [ diff --git a/research/selected-final-stage-portability/VERIFY.json b/research/selected-final-stage-portability/VERIFY.json index 621c8059b0..91583a6c0c 100644 --- a/research/selected-final-stage-portability/VERIFY.json +++ b/research/selected-final-stage-portability/VERIFY.json @@ -6,6 +6,7 @@ "inputs": [ "research/community-frontier1042-audit/RESULT.json", "research/community-frontier1042-audit/selected-replay.tar.gz", + "research/community-frontier1094-audit/ASSEMBLY.json", "research/community-frontier1094-audit/REPLAY.json", "research/community-frontier1094-audit/RESULT.json", "research/community-frontier1094-audit/STAGES.json", @@ -21,7 +22,7 @@ "{package}/verify.py" ], "timeout_seconds": 300, - "checks": "Binds slots/local to the #1042 receipt and the archived reviewed output to the #1094 receipts; reruns the final stage; requires 102 byte-identical, 1 json-relocated and 2 seconds-only files; six negative controls (proof value, kappa, gzip payload, unmapped path flagged; missing file reported; re-compressed gzip accepted)." + "checks": "Binds slots/local to the #1042 receipt and the archived reviewed output to the #1094 receipts; reruns the final stage; requires the rerun ASSEMBLY.json to equal the committed #1094 certificate bytes and 102 byte-identical, 1 json-relocated and 2 seconds-only files; seven negative controls (proof value, kappa, inner log digest, gzip payload, unmapped path flagged; missing file reported; re-compressed gzip accepted)." } ] } diff --git a/research/selected-final-stage-portability/compare_payloads.py b/research/selected-final-stage-portability/compare_payloads.py index 88f6a3ce90..a2a2aea73b 100644 --- a/research/selected-final-stage-portability/compare_payloads.py +++ b/research/selected-final-stage-portability/compare_payloads.py @@ -9,8 +9,9 @@ directories are replaced by the same placeholder, in keys and string values), ``missing`` / ``extra``. A remaining JSON difference is reported as ``volatile`` only when every differing leaf is under a key named in VOLATILE_KEYS (wall-clock -seconds and the digests of logs that print them); everything else is reported -as ``proof-relevant-candidate`` for a human to inspect. +``seconds``; a ``--volatile-key`` option adds more, e.g. ``log_sha256`` when the +logs print timings and are compared as files anyway); everything else is +reported as ``proof-relevant-candidate`` for a human to inspect. Standard library only; Python 3.10+. Prepared with Claude Code (Claude Opus 5.5) for JosieElliston's donated session 26. Apache-2.0. @@ -21,7 +22,7 @@ import json from pathlib import Path -VOLATILE_KEYS = frozenset({'seconds', 'log_sha256'}) +VOLATILE_KEYS = {'seconds'} def files(root): @@ -136,8 +137,11 @@ def main(): help='absolute directory used by the reference run, mapped to TOKEN') ap.add_argument('--run-root', action='append', default=[], metavar='TOKEN=PREFIX', help='absolute directory used by the compared run, mapped to TOKEN') + ap.add_argument('--volatile-key', action='append', default=[], + help='additional JSON key whose differing values are classified volatile') ap.add_argument('--out', type=Path) a = ap.parse_args() + VOLATILE_KEYS.update(a.volatile_key) result = compare(a.ref, a.run, roots(a.ref_root), roots(a.run_root)) counts = {} for v in result.values(): diff --git a/research/selected-final-stage-portability/verify.py b/research/selected-final-stage-portability/verify.py index 2ec1a54a70..216bd3b49c 100644 --- a/research/selected-final-stage-portability/verify.py +++ b/research/selected-final-stage-portability/verify.py @@ -16,8 +16,9 @@ absolute paths, and ``REPLAY.json``/``STAGES.json`` differing only in wall-clock ``seconds``; 5. runs negative controls showing that the comparison flags a changed - proof-relevant value, a changed gzip payload, a missing file and an - unmapped path, and accepts a re-compressed gzip container. + proof-relevant value, a changed inner log digest, a changed gzip payload, + a missing file and an unmapped path, and accepts a re-compressed gzip + container. Standard library only; Python 3.10+ (``tarfile`` ``filter=`` needs 3.10.12+ or 3.11.4+). Prepared with Claude Code (Claude Opus 5.5) for JosieElliston's @@ -149,6 +150,9 @@ def run(work): check(res[name]['cls'] == cls, 'unexpected class for ' + name) for name in ('REPLAY.json', 'STAGES.json'): check(res[name]['verdict'] == 'volatile' and res[name]['differing'] == 11, name + ' differs beyond seconds') + # The #1094 certificate (selected at base 0b1cccf8) is this file's bytes. + check((out / 'strict/ASSEMBLY.json').read_bytes() == (A1094 / 'ASSEMBLY.json').read_bytes(), + 'rerun ASSEMBLY.json differs from the committed #1094 certificate bytes') print('PASS comparison', json.dumps(counts(res), sort_keys=True), flush=True) # 5. Negative controls on copies of the rerun output. controls(ref / 'candidate/strict', out / 'strict', ref_roots, run_roots, work / 'controls') @@ -183,6 +187,10 @@ def kappa(j): j['kappa_decimal'] = j['kappa_decimal'][:-1] + '9' variant('kappa-with-seconds', edit_json('REPLAY.json', kappa), 'REPLAY.json', 'json-diff:proof-relevant-candidate') + def log_digest(j): + j[0]['log_sha256'] = '0' * 64 + variant('inner-log-digest', edit_json('STAGES.json', log_digest), + 'STAGES.json', 'json-diff:proof-relevant-candidate') gz = 'global/hosted/GLOBAL-TEMPLATE.json.gz' def recompress(d):