diff --git a/.editorconfig b/.editorconfig index fd4b0710..2038d6fa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -58,8 +58,9 @@ end_of_line = lf end_of_line = lf # Python is CRLF by the `[*]` default (universal newlines; commonly edited on Windows). Pin LF -# only for a `.py` executed directly via its shebang, by path - here the CI validation entry point. -[spec/validate.py] +# only for a `.py` executed directly via its shebang, by path - here the CI validation entry point +# and the fleet-audit runner. +[spec/{validate,audit}.py] end_of_line = lf # uv regenerates uv.lock with LF on every platform, so pin it or an EOL check (editorconfig-checker/CI) diff --git a/.gitattributes b/.gitattributes index abd48765..126849d2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -17,6 +17,7 @@ catalog/snippets/husky/pre-commit text eol=lf # commonly edited on Windows. Pin LF only for a `.py` executed directly via its shebang, by path - # here the CI validation entry point; do not re-add a blanket `*.py text eol=lf`. spec/validate.py text eol=lf +spec/audit.py text eol=lf # uv regenerates uv.lock with LF on every platform; pin it so git enforces LF on checkout/renormalize and a # CRLF-default repo does not fight the tool on every `uv lock`/`uv sync`. A repo with no lockfile is unaffected. diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 0f6c6159..951c7ec1 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -42,11 +42,11 @@ jobs: with: dotnet-version: 10.x - # Full history so NBGV can compute the git height for the branch. + # Full history for NBGV; pin the dispatch-time commit - a push landing after dispatch must not release unvalidated. - name: Checkout code step uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ github.ref_name }} + ref: ${{ github.sha }} fetch-depth: 0 # NBGV versions the dispatched ref: main is the public-release ref (clean X.Y.Z), develop a prerelease. diff --git a/AUDIT.md b/AUDIT.md index 52752a18..430fd800 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -10,7 +10,7 @@ The verdict vocabulary is [`WORKFLOW.md`][workflow]'s: **operational / not opera This audit is not occasional. Run it whenever you **create, adopt, or materially change** a fleet repo, and on demand for any known repo: - **Onboarding a repo is complete only when it either passes this audit** (operational - every applicable check) **or carries a committed `reports//audit.md` plus a tracking issue** enumerating every residual delta. A repo that is partially set up but never audited is itself a **defect** - the exact state this process prevents. The create-to-conformance counterpart is [`STANDUP.md`][standup]; because both read the same manifests, a repo stood up by that file passes this audit by construction. -- **Touching a repo** (any conformance-affecting change) ends by re-running the applicable checks and **reconciling the registry entry to reality** - `status`, `types`, `releaseTrigger`, `workflowModel`, `driftNotes`. The registry records reality, not intent; [`spec/validate.py`][validate] proves the catalog is self-consistent, not that it matches the live repo - closing that gap is this audit's job. +- **Touching a repo** (any conformance-affecting change) ends by re-running the applicable checks and **reconciling the registry entry to reality** - `status`, `types`, `releaseTrigger`, `workflowModel`, `driftNotes`. The registry records reality, not intent; [`spec/validate.py`][validate] proves the catalog is self-consistent, not that it matches the live repo - closing that gap is this audit's job. The deterministic subset (settings, rulesets, secret names, file presence, branch facts) is mechanized in [`spec/audit.py`][audit-runner]: owner-initiated, run on demand when onboarding a repo, on suspected drift, or before fleet-wide changes. ## 1. Scope and Ground-Truth Branch @@ -114,6 +114,7 @@ The convergence model: the hub audits and the agent **applies** the fixes via ta [agents]: ./AGENTS.md +[audit-runner]: ./spec/audit.py [codestyle]: ./CODESTYLE.md [copilot-runbook]: ./.github/copilot-instructions.md [project-types]: ./spec/project-types.json diff --git a/catalog/snippets/configs/vscode-tasks-python.json b/catalog/snippets/configs/vscode-tasks-python.json index d3d97656..593f1078 100644 --- a/catalog/snippets/configs/vscode-tasks-python.json +++ b/catalog/snippets/configs/vscode-tasks-python.json @@ -1,7 +1,8 @@ { "version": "2.0.0", "tasks": [ - // Python language group. Every task is `type: process` (like the .NET snippet) so the command is + // Python language group. Every command-executing task is `type: process` (like the .NET snippet; the + // aggregators are dependsOn-only) so the command is // executed directly, never through a shell - it avoids `&&`/`;` chaining, which is not portable // (Windows PowerShell 5.1, still the default VS Code task shell on many setups, rejects `&&`). // Sequencing is expressed with `dependsOrder: sequence` + `dependsOn`, not shell operators. @@ -64,7 +65,7 @@ "Ruff Check", "Python Types" ], - "group": "build", + "group": { "kind": "build", "isDefault": true }, "problemMatcher": [] }, { @@ -75,7 +76,7 @@ "run", "pytest" ], - "group": "test", + "group": { "kind": "test", "isDefault": true }, "problemMatcher": [], "presentation": { "showReuseMessage": false, diff --git a/repo-config/README.md b/repo-config/README.md index a6421bc9..413c0671 100644 --- a/repo-config/README.md +++ b/repo-config/README.md @@ -1,8 +1,8 @@ # repo-config -Repository and branch configuration held as committed files, kept out of `.github/` (which is reserved for GitHub-Actions-owned content). This mirrors the layout the fleet repos use. +Repository and branch configuration held as committed files, kept out of `.github/` (which holds the GitHub-consumed configuration - workflows, Dependabot). This mirrors the layout the fleet repos use. -- `main.json` plus one `develop` variant - the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos); the hub keeps both, a carried copy only its own model's (see "Downstream Carry"). These are the canonical expected payload the hub's audit (`AUDIT.md`, hub-only) diffs each repo's live rulesets against. +- `main.json` plus one `develop` variant - the branch rulesets as the writable API subset (`name`, `target`, `enforcement`, `bypass_actors`, `conditions`, `rules`). The `develop` payload is `develop.json` (`release` repos) or `operational/develop.json` (`operational` repos); the hub keeps both, a carried copy only its own model's (see "Downstream Carry"). These are the canonical expected payloads that the audit (the hub's fleet-wide `AUDIT.md`, or a carried repo-scoped adaptation - see "Downstream Carry") diffs the live rulesets against. - `operational/develop.json` - the `develop` ruleset for **operational** repos (registry `workflowModel: operational`): direct signed pushes, no PR gate. Present at the hub and in operational carries only - a carried `release` repo does not have it. See "Rulesets" below. - `configure.sh` - applies the rulesets to a repository via the GitHub API (create or full-payload update, idempotent). Run `repo-config/configure.sh [owner/repo] [release|operational]`; the model defaults to the registry `workflowModel` lookup. @@ -11,7 +11,8 @@ Repository and branch configuration held as committed files, kept out of `.githu Every fleet repo carries this directory; the hub keeps the canonical copy. Rules for the carried copy: - **Carry only your model's `develop` variant.** A `release` repo carries `develop.json`; an `operational` repo carries `operational/develop.json` instead. `main.json` and `settings.json` are shared by both models. `configure.sh` aborts when the payload its model needs is missing rather than applying a partial configuration. -- **Hub-only references stay plain text.** The hub is a private repo: never URL-link it from a downstream repo - the link 404s for anyone without hub access. Files that exist only at the hub (`AUDIT.md`, `spec/`) are mentioned by name, not linked; links into files every repo carries (`AGENTS.md`) resolve everywhere and are fine. +- **Hub-only references stay plain text.** The hub is a private repo: never URL-link it from a downstream repo - the link 404s for anyone without hub access. Files whose canonical fleet-wide form lives only at the hub are mentioned by name, not linked; links into files every repo carries (`AGENTS.md`) resolve everywhere and are fine. +- **Adapted self-audit carry.** A downstream repo carries **locally adapted** `AUDIT.md` and `spec/secrets.json`, scoped to self-auditing its own rulesets, settings, and secrets against the committed `repo-config/` baseline - the standard shape, so the carried tooling is self-contained. The hub's fleet-wide audit remains authoritative, and the local copies never link the hub. - **The regen snippet targets the current repo**, so it works unchanged in a carried copy. ## Rulesets @@ -42,7 +43,7 @@ done ## Secrets -Publish credentials required per mechanism are enumerated in the hub's `spec/secrets.json` (hub-only). A repo needs only the mechanisms its own publish targets use - a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. +Publish credentials required per mechanism are enumerated in `spec/secrets.json` (canonical at the hub; a downstream repo carries a repo-scoped adaptation - see "Downstream Carry"). A repo needs only the mechanisms its own publish targets use - a source-only repo needs none of the publish credentials below. NuGet and PyPI use keyless OIDC Trusted Publishing (no stored key; the publish job needs `id-token: write`, and PyPI additionally an `environment: pypi` gate). Docker Hub has no OIDC equivalent and uses a stored `DOCKER_HUB_USERNAME` + `DOCKER_HUB_ACCESS_TOKEN` in both the Actions and Dependabot secret stores. Codegen and merge-bot repos add a GitHub App (`CODEGEN_APP_CLIENT_ID` + `CODEGEN_APP_PRIVATE_KEY` in both stores; the app must be installed, not just created). App-token call sites use `client-id`, never the deprecated `app-id`. ## Repo Settings diff --git a/spec/audit.py b/spec/audit.py new file mode 100644 index 00000000..be05929a --- /dev/null +++ b/spec/audit.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Live fleet audit: the deterministic subset of AUDIT.md, run from the hub (stdlib + gh). + +Compares each cataloged registry repo against the ground truth in this repo - general settings +(repo-config/settings.json), branch rulesets (normalized diff vs the model's payloads), secret +names (spec/secrets.json; values are never read), baseline/per-type file presence on the +ground-truth branch (spec/files.json), and branch-model facts (main/develop existence, develop +behind main). Owner-initiated: run it when onboarding a repo, when drift is suspected, or before +fleet-wide changes. Read-only - it never modifies a target. + +Findings: DEFECT (an applicable check fails outright), LETTER (a required file is absent - intent +unverified, judge per AUDIT.md section 7), DRIFT (non-breaking divergence, e.g. main carrying +content develop lacks, a stale secret, a registry field contradicting reality), ERROR (a gh call +failed, so the repo could not be fully audited). Exits non-zero when any repo has a DEFECT, +LETTER, or ERROR finding. + +Usage: python3 spec/audit.py [RepoName ...] (default: every cataloged repo) +""" +import json +import pathlib +import subprocess +import sys + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +SETTINGS_KEYS = [ + "has_wiki", "has_projects", "allow_merge_commit", "allow_squash_merge", + "allow_rebase_merge", "allow_auto_merge", "allow_update_branch", "delete_branch_on_merge", +] +RULESET_SUBSET = ["name", "target", "enforcement", "bypass_actors", "conditions", "rules"] + + +def load(rel): + return json.loads((ROOT / rel).read_text(encoding="utf-8")) + + +def gh(path, ok404=False): + """GET a REST path via gh; parsed JSON, or None on 404 when ok404. + + No --paginate: on object endpoints it concatenates page documents into unparseable JSON. Every + list read here fits one page; callers pass per_page=100 where a default page could truncate. + """ + r = subprocess.run(["gh", "api", path], capture_output=True, text=True) + if r.returncode != 0: + if ok404 and ("HTTP 404" in r.stderr or "Not Found" in r.stderr): + return None + raise RuntimeError(f"gh api {path}: {r.stderr.strip().splitlines()[-1] if r.stderr else 'failed'}") + return json.loads(r.stdout) if r.stdout.strip() else None + + +def normalize_ruleset(payload): + sub = {k: payload.get(k) for k in RULESET_SUBSET} + if isinstance(sub.get("rules"), list): + sub["rules"] = sorted(sub["rules"], key=lambda r: json.dumps(r, sort_keys=True)) + if isinstance(sub.get("bypass_actors"), list): + sub["bypass_actors"] = sorted(sub["bypass_actors"], key=lambda a: (str(a.get("actor_type")), str(a.get("actor_id")))) + return json.dumps(sub, sort_keys=True) + + +def repo_slug(entry): + # url is https://github.com// + return "/".join(entry["url"].rstrip("/").split("/")[-2:]) + + +def audit_repo(entry, spec): + findings = [] # (kind, text) + slug = repo_slug(entry) + types = entry.get("types", []) + model = entry.get("workflowModel") or spec["registry"].get("defaults", {}).get("workflowModel") or "release" + ground = entry.get("groundTruthBranch", "main") + + try: + live = gh(f"repos/{slug}") + except RuntimeError as e: + return [("ERROR", str(e))] + + # --- Branch facts --- + branch_main = gh(f"repos/{slug}/branches/main", ok404=True) + branch_dev = gh(f"repos/{slug}/branches/develop", ok404=True) + main_exists, dev_exists = branch_main is not None, branch_dev is not None + if not main_exists: + findings.append(("DEFECT", "branch: main does not exist")) + if bool(entry.get("hasDevelop")) != dev_exists: + findings.append(("DRIFT", f"registry: hasDevelop={entry.get('hasDevelop')} but develop {'exists' if dev_exists else 'is absent'}")) + if main_exists and dev_exists: + # Commit counts mislead here: merge-commit promotions leave main permanently "ahead" while the + # trees are identical. Content is the signal - a develop...main compare with changed files means + # main carries content develop lacks (forward-sync needed); develop merely ahead is normal. + if branch_main["commit"]["commit"]["tree"]["sha"] != branch_dev["commit"]["commit"]["tree"]["sha"]: + cmp = gh(f"repos/{slug}/compare/develop...main", ok404=True) + if cmp and cmp.get("files"): + findings.append(("DRIFT", f"branch: main carries {len(cmp['files'])}+ changed file(s) develop lacks (forward-sync needed)")) + + # --- General settings --- + expected = dict(spec["settings"]) + expected["has_discussions"] = not live.get("private", True) + for key in SETTINGS_KEYS + ["has_discussions"]: + if key in expected and live.get(key) != expected[key]: + findings.append(("DEFECT", f"settings: {key} live={live.get(key)} expected={expected[key]}")) + if main_exists and live.get("default_branch") != "main": + findings.append(("DEFECT", f"settings: default_branch is {live.get('default_branch')}, expected main")) + + # --- Rulesets --- + dev_payload = "repo-config/operational/develop.json" if model == "operational" else "repo-config/develop.json" + expect_rulesets = {"develop": load(dev_payload), "main": load("repo-config/main.json")} + # No ok404: a repo with no rulesets returns an empty list, so a 404 means the call failed + # (access/rename) and must surface as ERROR, not read as "no rulesets". + live_list = gh(f"repos/{slug}/rulesets?per_page=100") or [] + live_names = [r["name"] for r in live_list] + for name, payload in expect_rulesets.items(): + ids = [r["id"] for r in live_list if r["name"] == name] + if not ids: + findings.append(("DEFECT", f"ruleset: {name} missing")) + continue + if len(ids) > 1: + findings.append(("DRIFT", f"ruleset: {len(ids)} rulesets named {name} (resolve the duplicates)")) + live_rs = gh(f"repos/{slug}/rulesets/{ids[0]}") + if normalize_ruleset(live_rs) != normalize_ruleset(payload): + findings.append(("DEFECT", f"ruleset: {name} diverges from {dev_payload if name == 'develop' else 'repo-config/main.json'} (normalized diff)")) + for stray in [n for n in live_names if n not in expect_rulesets]: + findings.append(("DRIFT", f"ruleset: stray ruleset '{stray}'")) + + # --- Secrets (names only) --- + secrets = spec["secrets"] + stores = {} + # No ok404: an empty store returns {"secrets": []}, so a 404/403 (permissions, rename) must + # surface as ERROR rather than cascade into false missing-secret DEFECTs. + for store, path in [("actions", f"repos/{slug}/actions/secrets?per_page=100"), ("dependabot", f"repos/{slug}/dependabot/secrets?per_page=100")]: + data = gh(path) + stores[store] = {s["name"] for s in (data or {}).get("secrets", [])} + mechanisms = [secrets["targetMechanisms"].get(p.get("target")) for p in entry.get("publish", [])] + mechanisms += [secrets.get("typeMechanisms", {}).get(t) for t in types] + claimed = [secrets["mechanisms"][m] for m in mechanisms if m and m in secrets["mechanisms"]] + required_by_store = {"actions": set(), "dependabot": set()} + for store in secrets["baseline"].get("stores", []): + required_by_store[store] |= set(secrets["baseline"].get("requires", [])) + for mech in claimed: + for store in mech.get("stores", []): + required_by_store[store] |= set(mech.get("requires", [])) + # Registry requiredSecrets[] are the domain-specific additions (STANDUP.md: requiredSecrets plus the + # implicit baseline). Mechanism-mapped names already carry their stores above; unmapped ones are + # expected in the actions store and count as claimed (never stale). + required_by_store["actions"] |= set(entry.get("requiredSecrets", [])) + forbidden = set(secrets["baseline"].get("forbids", [])) + for mech in claimed: + forbidden |= set(mech.get("forbids", [])) + for store, required in required_by_store.items(): + for name in sorted(required - stores[store]): + findings.append(("DEFECT", f"secrets: {name} missing from the {store} store")) + for store, present in stores.items(): + for name in sorted(present & forbidden): + findings.append(("DEFECT", f"secrets: forbidden {name} present in the {store} store")) + claimed_names = required_by_store["actions"] | required_by_store["dependabot"] + for name in sorted(present - claimed_names): + findings.append(("DRIFT", f"secrets: {name} in the {store} store is claimed by no applicable mechanism (stale?)")) + + # --- File presence on the ground-truth branch --- + seen_paths = set() + for item in spec["files"]["baseline"]: + applies = item.get("appliesTo", "*") + if applies != "*" and not set(applies) & set(types): + continue + path = item["path"] + if path == "repo-config/develop.json" and model == "operational": + path = "repo-config/operational/develop.json" + if path in seen_paths: + continue + seen_paths.add(path) + if gh(f"repos/{slug}/contents/{path}?ref={ground}", ok404=True) is None: + findings.append(("LETTER", f"file: {path} absent on {ground} (verify intent per AUDIT.md section 7)")) + + return findings + + +def main(): + spec = { + "registry": load("registry/repos.json"), + "settings": load("repo-config/settings.json"), + "secrets": load("spec/secrets.json"), + "files": load("spec/files.json"), + } + wanted = {n.lower() for n in sys.argv[1:]} + repos = [r for r in spec["registry"]["repos"] if r.get("status") == "cataloged"] + if wanted: + repos = [r for r in repos if r["name"].lower() in wanted] + missing = wanted - {r["name"].lower() for r in repos} + if missing: + print(f"Not cataloged: {', '.join(sorted(missing))}", file=sys.stderr) + return 2 + + hard = 0 + for entry in repos: + model = entry.get("workflowModel") or spec["registry"].get("defaults", {}).get("workflowModel") or "release" + print(f"== {entry['name']} ({', '.join(entry.get('types', []))}; {model}) ==") + try: + findings = audit_repo(entry, spec) + except Exception as e: # a gh/JSON failure mid-audit must not abort the sweep + findings = [("ERROR", str(e))] + if not findings: + print(" clean (deterministic checks; the full operational verdict is AUDIT.md's)") + for kind, text in findings: + print(f" {kind:6} {text}") + if kind in ("DEFECT", "LETTER", "ERROR"): + hard += 1 + print(f"\n{len(repos)} repo(s) audited; {hard} defect/letter/error finding(s).") + return 1 if hard else 0 + + +if __name__ == "__main__": + sys.exit(main())