Add spec/audit.py: on-demand live fleet audit (roadmap #310 slice D)

.editorconfig

@@ -58,8 +58,9 @@ end_of_line = lf
 end_of_line = lf
 
 # Python is CRLF by the `[*]` default (universal newlines; commonly edited on Windows). Pin LF
-# only for a `.py` executed directly via its shebang, by path - here the CI validation entry point.
-[spec/validate.py]
+# only for a `.py` executed directly via its shebang, by path - here the CI validation entry point
+# and the fleet-audit runner.
+[spec/{validate,audit}.py]
 end_of_line = lf
 
 # uv regenerates uv.lock with LF on every platform, so pin it or an EOL check (editorconfig-checker/CI)

.gitattributes

@@ -17,6 +17,7 @@ catalog/snippets/husky/pre-commit text eol=lf
 # commonly edited on Windows. Pin LF only for a `.py` executed directly via its shebang, by path -
 # here the CI validation entry point; do not re-add a blanket `*.py text eol=lf`.
 spec/validate.py text eol=lf
+spec/audit.py text eol=lf
 
 # uv regenerates uv.lock with LF on every platform; pin it so git enforces LF on checkout/renormalize and a
 # CRLF-default repo does not fight the tool on every `uv lock`/`uv sync`. A repo with no lockfile is unaffected.

AUDIT.md

@@ -10,7 +10,7 @@ The verdict vocabulary is [`WORKFLOW.md`][workflow]'s: **operational / not opera
 This audit is not occasional. Run it whenever you **create, adopt, or materially change** a fleet repo, and on demand for any known repo:
 
 - **Onboarding a repo is complete only when it either passes this audit** (operational - every applicable check) **or carries a committed `reports/<repo>/audit.md` plus a tracking issue** enumerating every residual delta. A repo that is partially set up but never audited is itself a **defect** - the exact state this process prevents. The create-to-conformance counterpart is [`STANDUP.md`][standup]; because both read the same manifests, a repo stood up by that file passes this audit by construction.
-- **Touching a repo** (any conformance-affecting change) ends by re-running the applicable checks and **reconciling the registry entry to reality** - `status`, `types`, `releaseTrigger`, `workflowModel`, `driftNotes`. The registry records reality, not intent; [`spec/validate.py`][validate] proves the catalog is self-consistent, not that it matches the live repo - closing that gap is this audit's job.
+- **Touching a repo** (any conformance-affecting change) ends by re-running the applicable checks and **reconciling the registry entry to reality** - `status`, `types`, `releaseTrigger`, `workflowModel`, `driftNotes`. The registry records reality, not intent; [`spec/validate.py`][validate] proves the catalog is self-consistent, not that it matches the live repo - closing that gap is this audit's job. The deterministic subset (settings, rulesets, secret names, file presence, branch facts) is mechanized in [`spec/audit.py`][audit-runner]: owner-initiated, run on demand when onboarding a repo, on suspected drift, or before fleet-wide changes.
 
 ## 1. Scope and Ground-Truth Branch
 
@@ -114,6 +114,7 @@ The convergence model: the hub audits and the agent **applies** the fixes via ta
 <!-- Repo -->
 
 [agents]: ./AGENTS.md
+[audit-runner]: ./spec/audit.py
 [codestyle]: ./CODESTYLE.md
 [copilot-runbook]: ./.github/copilot-instructions.md
 [project-types]: ./spec/project-types.json

spec/audit.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub (stdlib + gh).
+
+Compares each cataloged registry repo against the ground truth in this repo - general settings
+(repo-config/settings.json), branch rulesets (normalized diff vs the model's payloads), secret
+names (spec/secrets.json; values are never read), baseline/per-type file presence on the
+ground-truth branch (spec/files.json), and branch-model facts (main/develop existence, develop
+behind main). Owner-initiated: run it when onboarding a repo, when drift is suspected, or before
+fleet-wide changes. Read-only - it never modifies a target.
+
+Findings: DEFECT (an applicable check fails outright), LETTER (a required file is absent - intent
+unverified, judge per AUDIT.md section 7), DRIFT (non-breaking divergence, e.g. main carrying
+content develop lacks, a stale secret, a registry field contradicting reality), ERROR (a gh call
+failed, so the repo could not be fully audited). Exits non-zero when any repo has a DEFECT,
+LETTER, or ERROR finding.
+
+Usage: python3 spec/audit.py [RepoName ...]   (default: every cataloged repo)
+"""
+import json
+import pathlib
+import subprocess
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+
+SETTINGS_KEYS = [
+    "has_wiki", "has_projects", "allow_merge_commit", "allow_squash_merge",
+    "allow_rebase_merge", "allow_auto_merge", "allow_update_branch", "delete_branch_on_merge",
+]
+RULESET_SUBSET = ["name", "target", "enforcement", "bypass_actors", "conditions", "rules"]
+
+
+def load(rel):
+    return json.loads((ROOT / rel).read_text(encoding="utf-8"))
+
+
+def gh(path, ok404=False):
+    """GET a REST path via gh; parsed JSON, or None on 404 when ok404.
+
+    No --paginate: on object endpoints it concatenates page documents into unparseable JSON. Every
+    list read here fits one page; callers pass per_page=100 where a default page could truncate.
+    """
+    r = subprocess.run(["gh", "api", path], capture_output=True, text=True)
+    if r.returncode != 0:
+        if ok404 and ("HTTP 404" in r.stderr or "Not Found" in r.stderr):
+            return None
+        raise RuntimeError(f"gh api {path}: {r.stderr.strip().splitlines()[-1] if r.stderr else 'failed'}")
+    return json.loads(r.stdout) if r.stdout.strip() else None
+
+
+def normalize_ruleset(payload):
+    sub = {k: payload.get(k) for k in RULESET_SUBSET}
+    if isinstance(sub.get("rules"), list):
+        sub["rules"] = sorted(sub["rules"], key=lambda r: json.dumps(r, sort_keys=True))
+    if isinstance(sub.get("bypass_actors"), list):
+        sub["bypass_actors"] = sorted(sub["bypass_actors"], key=lambda a: (str(a.get("actor_type")), str(a.get("actor_id"))))
+    return json.dumps(sub, sort_keys=True)
+
+
+def repo_slug(entry):
+    # url is https://github.com/<owner>/<repo>
+    return "/".join(entry["url"].rstrip("/").split("/")[-2:])
+
+
+def audit_repo(entry, spec):
+    findings = []  # (kind, text)
+    slug = repo_slug(entry)
+    types = entry.get("types", [])
+    model = entry.get("workflowModel") or spec["registry"].get("defaults", {}).get("workflowModel") or "release"
+    ground = entry.get("groundTruthBranch", "main")
+
+    try:
+        live = gh(f"repos/{slug}")
+    except RuntimeError as e:
+        return [("ERROR", str(e))]
+
+    # --- Branch facts ---
+    branch_main = gh(f"repos/{slug}/branches/main", ok404=True)
+    branch_dev = gh(f"repos/{slug}/branches/develop", ok404=True)
+    main_exists, dev_exists = branch_main is not None, branch_dev is not None
+    if not main_exists:
+        findings.append(("DEFECT", "branch: main does not exist"))
+    if bool(entry.get("hasDevelop")) != dev_exists:
+        findings.append(("DRIFT", f"registry: hasDevelop={entry.get('hasDevelop')} but develop {'exists' if dev_exists else 'is absent'}"))
+    if main_exists and dev_exists:
+        # Commit counts mislead here: merge-commit promotions leave main permanently "ahead" while the
+        # trees are identical. Content is the signal - a develop...main compare with changed files means
+        # main carries content develop lacks (forward-sync needed); develop merely ahead is normal.
+        if branch_main["commit"]["commit"]["tree"]["sha"] != branch_dev["commit"]["commit"]["tree"]["sha"]:
+            cmp = gh(f"repos/{slug}/compare/develop...main", ok404=True)
+            if cmp and cmp.get("files"):
+                findings.append(("DRIFT", f"branch: main carries {len(cmp['files'])}+ changed file(s) develop lacks (forward-sync needed)"))
+
+    # --- General settings ---
+    expected = dict(spec["settings"])
+    expected["has_discussions"] = not live.get("private", True)
+    for key in SETTINGS_KEYS + ["has_discussions"]:
+        if key in expected and live.get(key) != expected[key]:
+            findings.append(("DEFECT", f"settings: {key} live={live.get(key)} expected={expected[key]}"))
+    if main_exists and live.get("default_branch") != "main":
+        findings.append(("DEFECT", f"settings: default_branch is {live.get('default_branch')}, expected main"))
+
+    # --- Rulesets ---
+    dev_payload = "repo-config/operational/develop.json" if model == "operational" else "repo-config/develop.json"
+    expect_rulesets = {"develop": load(dev_payload), "main": load("repo-config/main.json")}
+    live_list = gh(f"repos/{slug}/rulesets?per_page=100", ok404=True) or []
+    live_names = [r["name"] for r in live_list]
+    for name, payload in expect_rulesets.items():
+        ids = [r["id"] for r in live_list if r["name"] == name]
+        if not ids:
+            findings.append(("DEFECT", f"ruleset: {name} missing"))
+            continue
+        if len(ids) > 1:
+            findings.append(("DRIFT", f"ruleset: {len(ids)} rulesets named {name} (resolve the duplicates)"))
+        live_rs = gh(f"repos/{slug}/rulesets/{ids[0]}")
+        if normalize_ruleset(live_rs) != normalize_ruleset(payload):
+            findings.append(("DEFECT", f"ruleset: {name} diverges from {dev_payload if name == 'develop' else 'repo-config/main.json'} (normalized diff)"))
+    for stray in [n for n in live_names if n not in expect_rulesets]:
+        findings.append(("DRIFT", f"ruleset: stray ruleset '{stray}'"))
+
+    # --- Secrets (names only) ---
+    secrets = spec["secrets"]
+    stores = {}
+    for store, path in [("actions", f"repos/{slug}/actions/secrets?per_page=100"), ("dependabot", f"repos/{slug}/dependabot/secrets?per_page=100")]:
+        data = gh(path, ok404=True)
+        stores[store] = {s["name"] for s in (data or {}).get("secrets", [])}
+    mechanisms = [secrets["targetMechanisms"].get(p.get("target")) for p in entry.get("publish", [])]
+    mechanisms += [secrets.get("typeMechanisms", {}).get(t) for t in types]
+    claimed = [secrets["mechanisms"][m] for m in mechanisms if m and m in secrets["mechanisms"]]
+    required_by_store = {"actions": set(), "dependabot": set()}
+    for store in secrets["baseline"].get("stores", []):
+        required_by_store[store] |= set(secrets["baseline"].get("requires", []))
+    for mech in claimed:
+        for store in mech.get("stores", []):
+            required_by_store[store] |= set(mech.get("requires", []))
+    # Registry requiredSecrets[] are the domain-specific additions (STANDUP.md: requiredSecrets plus the
+    # implicit baseline). Mechanism-mapped names already carry their stores above; unmapped ones are
+    # expected in the actions store and count as claimed (never stale).
+    required_by_store["actions"] |= set(entry.get("requiredSecrets", []))
+    forbidden = set(secrets["baseline"].get("forbids", []))
+    for mech in claimed:
+        forbidden |= set(mech.get("forbids", []))
+    for store, required in required_by_store.items():
+        for name in sorted(required - stores[store]):
+            findings.append(("DEFECT", f"secrets: {name} missing from the {store} store"))
+    for store, present in stores.items():
+        for name in sorted(present & forbidden):
+            findings.append(("DEFECT", f"secrets: forbidden {name} present in the {store} store"))
+        claimed_names = required_by_store["actions"] | required_by_store["dependabot"]
+        for name in sorted(present - claimed_names):
+            findings.append(("DRIFT", f"secrets: {name} in the {store} store is claimed by no applicable mechanism (stale?)"))
+
+    # --- File presence on the ground-truth branch ---
+    seen_paths = set()
+    for item in spec["files"]["baseline"]:
+        applies = item.get("appliesTo", "*")
+        if applies != "*" and not set(applies) & set(types):
+            continue
+        path = item["path"]
+        if path == "repo-config/develop.json" and model == "operational":
+            path = "repo-config/operational/develop.json"
+        if path in seen_paths:
+            continue
+        seen_paths.add(path)
+        if gh(f"repos/{slug}/contents/{path}?ref={ground}", ok404=True) is None:
+            findings.append(("LETTER", f"file: {path} absent on {ground} (verify intent per AUDIT.md section 7)"))
+
+    return findings
+
+
+def main():
+    spec = {
+        "registry": load("registry/repos.json"),
+        "settings": load("repo-config/settings.json"),
+        "secrets": load("spec/secrets.json"),
+        "files": load("spec/files.json"),
+    }
+    wanted = {n.lower() for n in sys.argv[1:]}
+    repos = [r for r in spec["registry"]["repos"] if r.get("status") == "cataloged"]
+    if wanted:
+        repos = [r for r in repos if r["name"].lower() in wanted]
+        missing = wanted - {r["name"].lower() for r in repos}
+        if missing:
+            print(f"Not cataloged: {', '.join(sorted(missing))}", file=sys.stderr)
+            return 2
+
+    hard = 0
+    for entry in repos:
+        model = entry.get("workflowModel") or spec["registry"].get("defaults", {}).get("workflowModel") or "release"
+        print(f"== {entry['name']} ({', '.join(entry.get('types', []))}; {model}) ==")
+        try:
+            findings = audit_repo(entry, spec)
+        except Exception as e:  # a gh/JSON failure mid-audit must not abort the sweep
+            findings = [("ERROR", str(e))]
+        if not findings:
+            print("  clean (deterministic checks; the full operational verdict is AUDIT.md's)")
+        for kind, text in findings:
+            print(f"  {kind:6} {text}")
+            if kind in ("DEFECT", "LETTER", "ERROR"):
+                hard += 1
+    print(f"\n{len(repos)} repo(s) audited; {hard} defect/letter/error finding(s).")
+    return 1 if hard else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())