diff --git a/.editorconfig b/.editorconfig index 464d489c..bb2a1382 100644 --- a/.editorconfig +++ b/.editorconfig @@ -60,7 +60,7 @@ end_of_line = lf # Python is CRLF by the `[*]` default (universal newlines; commonly edited on Windows). Pin LF # only for a `.py` executed directly via its shebang, by path - here the CI validation entry point # and the fleet-audit runner. -[spec/{validate,audit}.py] +[spec/{validate,audit,fidelity_honesty}.py] end_of_line = lf # The agent-safety kit's Python is shebang-executable tooling run by path (the PreToolUse hook and its diff --git a/.gitattributes b/.gitattributes index b69124f1..09fe1210 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,6 +19,7 @@ catalog/snippets/husky/pre-commit text eol=lf # installer. Do not re-add a blanket `*.py text eol=lf`. spec/validate.py text eol=lf spec/audit.py text eol=lf +spec/fidelity_honesty.py text eol=lf host-setup/agent-safety/gh-write-guard.py text eol=lf host-setup/agent-safety/install.py text eol=lf diff --git a/spec/fidelity_honesty.py b/spec/fidelity_honesty.py new file mode 100644 index 00000000..27d7d687 --- /dev/null +++ b/spec/fidelity_honesty.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Fidelity-honesty analysis: check the manifest's declared fidelities against fleet reality. + +Read-only, owner-run, not wired into CI. Reuses spec/audit.py's fleet machinery (gh, content_hash, +git history, selectors) - import-safe because audit.py guards its main. + +The verbatim engine verifies a unit's *content* against the canonical (declared -> hashed). This tool +verifies the *classifications themselves* (declared -> checked), the same declared-to-verified leap one +level up. It answers two questions the audit cannot: + + 1. Which `intent` units are actually content-identical (after EOL normalization) across the whole fleet? Those are candidates to + promote to `verbatim` - they would gain free drift-detection (a stale-but-present copy is invisible + under intent, caught under verbatim). This is the class that hid the configure.sh drift. + 2. Which `verbatim` units have a downstream copy that diverges in a NON-stale way? That is either a + mis-set label (the content legitimately varies -> should be intent) or real drift to chase. + +It also runs a manifest-gap pass: a file present in BOTH the hub and a reference adopter but absent from +the manifest is carried-but-untracked (exactly the configure.sh / settings.json bug). + +Usage: python3 spec/fidelity_honesty.py [reference-repo-for-manifest-gap] (default: Financial-Modeling) +""" +import base64 +import subprocess +import sys + +import audit # sibling, import-safe (its main is guarded) + +REF_ADOPTER = "Financial-Modeling" # a well-adopted repo, used only for the manifest-gap pass + + +def canonical_text(entry): + """The hub's canonical for a unit: its reference snippet, else its own root copy.""" + ref = entry.get("reference") or entry["path"] + try: + return (audit.ROOT / ref).read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def fetch(slug, path, ref): + """Decoded downstream file content, or None if absent / not inline.""" + content = audit.gh(f"repos/{slug}/contents/{path}?ref={ref}", ok404=True) + if content is None or content.get("encoding") != "base64": + return None + return base64.b64decode(content["content"]).decode("utf-8", "replace") + + +def fidelity_pass(spec): + defaults = spec["registry"].get("defaults", {}) + repos = [r for r in spec["registry"]["repos"] if r.get("status") == "cataloged"] + units = [e for e in spec["files"]["baseline"] if e.get("fidelity") in ("intent", "verbatim")] + + spreads = [] # (entry, spread dict) for the full table + promote = [] # intent units that are uniform fleet-wide + mislabel = [] # verbatim units that diverge non-stale + + for e in units: + path, fid = e["path"], e["fidelity"] + canon = canonical_text(e) + if canon is None: + spreads.append((e, None)) + continue + canon_hash = audit.content_hash(canon) + history = {audit.content_hash(t) for t in audit.git_file_history(e.get("reference") or path)} + spread = {"match": [], "stale": [], "differs": [], "unavailable": []} + for r in repos: + if not audit.applies(e.get("appliesTo", "*"), audit.repo_selectors(r, defaults)): + continue + text = fetch(audit.repo_slug(r), path, r.get("groundTruthBranch", "main")) + if text is None: # missing (404) or present-but-non-inline (too large / encoding "none") + spread["unavailable"].append(r["name"]) + continue + dh = audit.content_hash(text) + if dh == canon_hash: + spread["match"].append(r["name"]) + elif dh in history: + spread["stale"].append(r["name"]) + else: + spread["differs"].append(r["name"]) + spreads.append((e, spread)) + # A verbatim candidate has NO hand-modified copy ("differs") and at least one confirmed match with + # the current canonical. Stale copies do not disqualify it - verbatim would flag them "stale -> + # re-vendor", which is the point. A unit that is entirely stale/unavailable is not confirmed uniform. + if fid == "intent" and spread["match"] and not spread["differs"]: + promote.append((e, spread)) + if fid == "verbatim" and spread["differs"]: + mislabel.append((e, spread)) + return spreads, promote, mislabel + + +def manifest_gap_pass(spec, ref_repo): + """Files present in BOTH the hub and the reference adopter but absent from the manifest.""" + listed = {e["path"] for e in spec["files"]["baseline"]} + entry = next((r for r in spec["registry"]["repos"] if r["name"] == ref_repo), None) + if entry is None: + return None, [] + slug = audit.repo_slug(entry) + ground = entry.get("groundTruthBranch", "main") + # The git/trees endpoint takes a tree SHA, not a ref name, so resolve the branch to its tree SHA + # first (as audit.py does) - passing the branch name can 404 and silently drop the whole check. + # Fail loud on an unreadable reference adopter: an empty gaps list would report "none" (a false clean). + br = audit.gh(f"repos/{slug}/branches/{ground}", ok404=True) + if not br or "commit" not in br: + raise RuntimeError(f"could not read {slug}@{ground} (missing branch?) - cannot run the manifest-gap pass") + tree = audit.gh(f"repos/{slug}/git/trees/{br['commit']['commit']['tree']['sha']}?recursive=1", ok404=True) + if not tree or "tree" not in tree: + raise RuntimeError(f"could not read the tree for {slug}@{ground} - cannot run the manifest-gap pass") + # The hub's tracked files (git ls-files), not a filesystem walk: a walk pulls in untracked local cruft + # (__pycache__, a local .venv) and would make the gap report depend on working-tree state. + r = subprocess.run(["git", "ls-files"], cwd=audit.ROOT, capture_output=True, text=True) + if r.returncode != 0: # fail loud: an empty set would masquerade as "no gaps" (a false clean) + raise RuntimeError(f"git ls-files failed in {audit.ROOT}: {r.stderr.strip() or 'non-zero exit'}") + hub_files = set(r.stdout.splitlines()) + gaps = sorted(n["path"] for n in tree["tree"] + if n.get("type") == "blob" and n["path"] in hub_files and n["path"] not in listed) + return slug, gaps + + +def main(): + ref_repo = sys.argv[1] if len(sys.argv) > 1 else REF_ADOPTER + spec = { + "registry": audit.load("registry/repos.json"), + "files": audit.load("spec/files.json"), + } + spreads, promote, mislabel = fidelity_pass(spec) + + print("== Per-unit fleet spread (ground-truth branch per repo) ==") + print(" fidelity path :: match / stale / differs / unavailable (absent or non-inline)") + for e, spread in spreads: + if spread is None: + print(f" {e.get('fidelity'):8} {e['path']} :: canonical unreadable at hub - skipped") + continue + print(f" {e['fidelity']:8} {e['path']} :: " + f"{len(spread['match'])} / {len(spread['stale'])} / {len(spread['differs'])} / {len(spread['unavailable'])}") + + print("\n== INTENT units with no divergent copy (verbatim-appropriate) -> candidates to promote to VERBATIM ==") + print(" (>=1 confirmed match, 0 hand-modified. Any stale/unavailable copy is shown per unit and would") + print(" re-vendor under verbatim - the drift intent cannot catch)") + if not promote: + print(" none") + for e, spread in promote: + print(f" {e['path']}: {len(spread['match'])} match, {len(spread['stale'])} stale, " + f"{len(spread['unavailable'])} unavailable, 0 differ") + + print("\n== VERBATIM units with NON-stale downstream divergence (mis-label or real drift) ==") + if not mislabel: + print(" none") + for e, spread in mislabel: + print(f" {e['path']}: differs in {', '.join(spread['differs'])}") + + print(f"\n== Manifest gap: files carried by {ref_repo} + present at the hub but NOT in the manifest ==") + slug, gaps = manifest_gap_pass(spec, ref_repo) + if slug is None: + print(f" {ref_repo} not found in the registry") + elif not gaps: + print(" none - the manifest covers every hub file the reference adopter also carries") + else: + for g in gaps: + print(f" UNTRACKED {g}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())