diff --git a/.editorconfig b/.editorconfig index 464d489c..bb2a1382 100644 --- a/.editorconfig +++ b/.editorconfig @@ -60,7 +60,7 @@ end_of_line = lf # Python is CRLF by the `[*]` default (universal newlines; commonly edited on Windows). Pin LF # only for a `.py` executed directly via its shebang, by path - here the CI validation entry point # and the fleet-audit runner. -[spec/{validate,audit}.py] +[spec/{validate,audit,fidelity_honesty}.py] end_of_line = lf # The agent-safety kit's Python is shebang-executable tooling run by path (the PreToolUse hook and its diff --git a/.gitattributes b/.gitattributes index b69124f1..09fe1210 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,6 +19,7 @@ catalog/snippets/husky/pre-commit text eol=lf # installer. Do not re-add a blanket `*.py text eol=lf`. spec/validate.py text eol=lf spec/audit.py text eol=lf +spec/fidelity_honesty.py text eol=lf host-setup/agent-safety/gh-write-guard.py text eol=lf host-setup/agent-safety/install.py text eol=lf diff --git a/CODESTYLE.md b/CODESTYLE.md index cd341e04..10902ddc 100644 --- a/CODESTYLE.md +++ b/CODESTYLE.md @@ -355,7 +355,7 @@ This is the style guide for any **Python project(s)** in this repo. **Two profiles.** A repo's Python is one of two shapes, and the rest of this section (uv project, `uv.lock`, `uv run`, `src` layout, pytest coverage) describes the **project** profile. The two differ by whether the Python has **third-party runtime dependencies**, which shows up structurally in `pyproject.toml`, so the audit detects the profile there (`python.profile.detect`): - **Project** - the Python has third-party runtime dependencies, or is the repo's deliverable. It is a PEP 621 uv project: `[project]` with `dependencies` (dev tools in `[project.optional-dependencies]` or `[dependency-groups]`), a `[build-system]`, and a committed `uv.lock` (pinned LF - see [Line Endings][line-endings]); CI runs `uv sync --frozen` + `uv run `, so the lockfile pins tool versions. -- **Scripts** - stdlib-only utility scripts embedded in a **non-Python** repo (e.g. a Python tooling subtree of a `csharp` app). Run the tools with **`uvx`** (no project install, no lockfile): the `pyproject.toml` carries **only** `[tool.ruff]` / `[tool.mypy]` config - no `[project]`, no `[build-system]`, no `uv.lock` (that metadata would misrepresent it as a shippable package). **mypy** is the type checker (there is no first-party package for pyright strict to anchor on). Because there is no lockfile to pin versions, **CI pins the exact tool versions in the `uvx` command** (`uvx ruff@`, `uvx mypy@`, bumpable there) while the VS Code tasks and README run the unpinned latest - a deliberate CI-vs-local gap so local tooling never silently falls behind. `.py` files follow the repo's line-ending default (CRLF in a CRLF-default repo; a shebang-executed script is LF-pinned by path - see [Line Endings][line-endings]). There is no pytest suite, so the coverage expectation is N/A; a co-present `csharp` type still carries `codecov.yml` for its own tests. +- **Scripts** - stdlib-only utility scripts embedded in a **non-Python** repo (e.g. a Python tooling subtree of a `csharp` app). Run the tools with **`uvx`** (no project install, no lockfile): the `pyproject.toml` carries **only** tool config (`[tool.ruff]`, `[tool.mypy]`, and an optional `[tool.pyright]` editor block) - no `[project]`, no `[build-system]`, no `uv.lock` (that metadata would misrepresent it as a shippable package). **mypy** is the type-check gate (there is no first-party package for pyright strict to anchor on), and a `[tool.pyright]` block in **standard** mode keeps Pylance quiet in the editor - the same mypy-gate/pyright-editor split the Project profile uses. There is no lockfile, and a `uvx @` pin in a `run:` step is not something Dependabot tracks, so **CI runs `uvx ruff@latest` / `uvx mypy@latest`** rather than a manual pin that would silently go stale. The fleet rule is to pin only what Dependabot auto-updates (SHA-pinned actions, package deps) and otherwise run latest, so the VS Code tasks, README, and CI all run the unpinned latest here. `.py` files follow the repo's line-ending default (CRLF in a CRLF-default repo, and a shebang-executed script is LF-pinned by path - see [Line Endings][line-endings]). There is no pytest suite, so the coverage expectation is N/A. A co-present `csharp` type still carries `codecov.yml` for its own tests. ### Toolchain diff --git a/WORKFLOW.md b/WORKFLOW.md index 57fe3748..9cb54dd6 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,8 +1,8 @@ # WORKFLOW.md -The guide for CI/CD **workflows** (GitHub Actions): a deliberate mixture of code style, architecture, a **behavioral contract** (expected inputs and outputs), and a **test methodology**. Code style lives in [`CODESTYLE.md`][codestyle]; this file is its sibling for everything under [`.github/workflows/`][workflows]. +The guide for CI/CD **workflows** (GitHub Actions): a deliberate mixture of code style, architecture, a **behavioral contract** (expected inputs and outputs), and a **test methodology**. Code style lives in [`CODESTYLE.md`][codestyle]. This file is its sibling for everything under [`.github/workflows/`][workflows]. -Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs** - not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible; the contract in section 4 is what they must *do*. +Its defining principle: **it describes required outcomes, not a required implementation.** Two repos may implement the same guarantee with different YAML. A workflow is correct when it **satisfies the contract** in section 4 and is **defect-free against the expected inputs and outputs** - not when it matches a reference implementation byte for byte. The conventions in section 2 keep workflows legible. The contract in section 4 is what they must *do*. Given this document, an agent must be able to do three things to any project: @@ -10,35 +10,35 @@ Given this document, an agent must be able to do three things to any project: 2. **Test** - trace the expected inputs/outputs (section 5B) and, where warranted, drive a live probe (section 5C). 3. **Assess** - render a verdict: **operational** (every *applicable* guarantee holds and every *applicable* scenario's observed output equals the expected) or **not operational** (any mismatch - which is a *defect*, not a style nit). -> **Canonical scope.** This document is authoritative for the workflow contract and test methodology (sections 3 to 6). The conventions in section 2 and the release policy also live in `AGENTS.md` ("Workflow YAML Conventions" and "Release Model"), which is authoritative where the two overlap; section 2 restates them so this file reads on its own. On any conflict in that overlap, `AGENTS.md` wins. +> **Canonical scope.** This document is authoritative for the workflow contract and test methodology (sections 3 to 6). The conventions in section 2 and the release policy also live in `AGENTS.md` ("Workflow YAML Conventions" and "Release Model"), which is authoritative where the two overlap. Section 2 restates them so this file reads on its own. On any conflict in that overlap, `AGENTS.md` wins. The guarantees are distilled from failures observed in practice and stated as the **failure-mode each prevents**, so the document stays portable to any project. ## 1. Purpose and How to Use This Document -- **Contract, not implementation.** Conform to the *outcomes* in section 4. Shape, job names, and file layout may differ between repos; the input/output behavior may not. -- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs - a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type; a near-empty pipeline (source-only) is mostly N/A and that is fine. +- **Contract, not implementation.** Conform to the *outcomes* in section 4. Shape, job names, and file layout may differ between repos, but the input/output behavior may not. +- **Applicability.** A guarantee (or a 5A check, or a 5B scenario) is **applicable** only if the repo contains the construct it governs - a given target, a transfer artifact, a registry push, a wrapper-version source. An item that governs an absent construct is **N/A**: record it as N/A and **exclude it from the verdict**. N/A is never a defect. Section 6 names which items go N/A per project type. A near-empty pipeline (source-only) is mostly N/A and that is fine. - **Operational is binary.** A workflow is operational only if every *applicable* guarantee holds. A single applicable input/output mismatch is a defect and makes the workflow non-operational, regardless of how clean the YAML looks. -- **Default branch.** Guarantees say "default branch" portably; it is implemented as the literal `main` in several places (the validate gate, the `prerelease` expression, and `version.json`'s `publicReleaseRefSpec`). These MUST all reference the repo's *actual* default branch; a divergence is a defect (section 5A). -- **Two layers when auditing.** The pipeline splits into an **orchestrator** layer (the PR entry workflow, the publisher, and the version/release/badge jobs) and a **build-leaf** layer (`build--task.yml`). Inputs like `github`/`nuget`/`dockerhub`/`expect_release_assets` live on the orchestrator; a leaf only ever receives `ref`/`branch`/`smoke` (and a derived `push`). When a check names an input, assert it in the layer that declares it. +- **Default branch.** Guarantees say "default branch" portably. It is implemented as the literal `main` in several places (the validate gate, the `prerelease` expression, and `version.json`'s `publicReleaseRefSpec`). These MUST all reference the repo's *actual* default branch. A divergence is a defect (section 5A). +- **Two layers when auditing.** The pipeline splits into an **orchestrator** layer (the PR entry workflow, the publisher, and the version/release/badge jobs) and a **build-leaf** layer (`build--task.yml`). Inputs like `github`/`nuget`/`dockerhub`/`expect_release_assets` live on the orchestrator. A leaf only ever receives `ref`/`branch`/`smoke` (and a derived `push`). When a check names an input, assert it in the layer that declares it. - **The three verbs.** Audit (static), Test (trace + probe), Assess (verdict). Section 5 gives the exact procedure. ## 2. Workflow Style Conventions Prescriptive style/legibility rules. Cheap to check, necessary but not sufficient (a perfectly styled workflow can still violate section 4). -- **Action pinning.** Pin **every** action to a commit SHA with a trailing `# vX.Y.Z` comment. Use `# vX` only when the upstream floating major tag has no specific patch SHA. The single documented no-pin exception is a tool whose tag stream lags `master` such that tag-tracking would downgrade (here, `dotnet/nbgv@master`); invent no others. -- **Filename.** Reusable workflows (`on: workflow_call`) end in `-task.yml`; entry-point workflows do not (`-pull-request.yml`, `-release.yml`). Lowercase, hyphen-separated. -- **Workflow `name:`.** Reusable names end in **"task"**; entry-point names end in **"action"**. +- **Action pinning.** Pin **every** action to a commit SHA with a trailing `# vX.Y.Z` comment. Use `# vX` only when the upstream floating major tag has no specific patch SHA. The single documented no-pin exception is a tool whose tag stream lags `master` such that tag-tracking would downgrade (here, `dotnet/nbgv@master`). Invent no others. +- **Filename.** Reusable workflows (`on: workflow_call`) end in `-task.yml`. Entry-point workflows do not (`-pull-request.yml`, `-release.yml`). Lowercase, hyphen-separated. +- **Workflow `name:`.** Reusable names end in **"task"**. Entry-point names end in **"action"**. - **Job and step `name:`.** Every job ends in **"job"**, every step in **"step"** - including a ruleset-bound required-check job, whose `name:` and the ruleset `context:` are one string renamed together (never independently). - **Concurrency.** Top-level workflows declare `concurrency: { group: '${{ github.workflow }}-${{ github.ref }}', cancel-in-progress: true }`. Document exceptions inline (D7). - **Shells.** Every multi-line bash `run:` - and every committed `.sh` script - starts `set -Eeuo pipefail`. - **Conditionals.** Multi-line `if:` uses the folded scalar `if: >-`. -- **Boolean inputs.** A boolean used by both `workflow_call` and `workflow_dispatch` is declared in **both** trigger blocks; `workflow_dispatch` delivers the **string** `"true"`/`"false"`, so any `if:` compares both forms: `${{ inputs.foo == true || inputs.foo == 'true' }}`. -- **Reusable-workflow permissions.** Job-level `permissions:` are validated **before** `if:`, so even a skipped job needs valid permissions. Grant least privilege; a reusable callee's extra scope (e.g. `actions: write` for cleanup) is granted by the **caller**. +- **Boolean inputs.** A boolean used by both `workflow_call` and `workflow_dispatch` is declared in **both** trigger blocks, and `workflow_dispatch` delivers the **string** `"true"`/`"false"`, so any `if:` compares both forms: `${{ inputs.foo == true || inputs.foo == 'true' }}`. +- **Reusable-workflow permissions.** Job-level `permissions:` are validated **before** `if:`, so even a skipped job needs valid permissions. Grant least privilege. A reusable callee's extra scope (e.g. `actions: write` for cleanup) is granted by the **caller**. - **Allowlist `success` and `skipped` explicitly** across optional dependencies (`!= 'failure'` lets `cancelled` through). - **Docker layer cache.** Cache to/from a registry tag (`type=registry`), never `type=gha`. -- **Line endings.** Workflow YAML is LF (Actions and Dependabot rewrite it that way); other files follow `.editorconfig`, and committed JSON state files follow the repo's JSON rule. Preserve endings on every edit. +- **Line endings.** Workflow YAML is LF (Actions and Dependabot rewrite it that way). Other files follow `.editorconfig`, and committed JSON state files follow the repo's JSON rule. Preserve endings on every edit. ## 3. Architecture @@ -71,7 +71,7 @@ Their CI is lint/validation only (editorconfig/EOL plus domain linters - Home As ### The Seam Contract -A target contributes a file to the GitHub release by uploading a workflow artifact named `release-asset--`. The release job collects **every** matching artifact by **pattern** (`pattern: release-asset--*` + `merge-multiple: true`), never an `artifact-ids:` naming one job's output. Canonical for **every** repo, single-target included; switching to an `artifact-id` handoff forks the release download and breaks the verbatim carry. +A target contributes a file to the GitHub release by uploading a workflow artifact named `release-asset--`. The release job collects **every** matching artifact by **pattern** (`pattern: release-asset--*` + `merge-multiple: true`), never an `artifact-ids:` naming one job's output. Canonical for **every** repo, single-target included. Switching to an `artifact-id` handoff forks the release download and breaks the verbatim carry. ```mermaid flowchart LR @@ -83,11 +83,11 @@ flowchart LR ### Reusable-Task Parameter Contract -Every leaf and the release task take `ref`, `branch` (the **logical** branch that drives config/tags/prerelease), and where relevant `smoke`. Branch-derived config keys off `inputs.branch` (the logical branch the caller passes); artifact names are branch-suffixed. +Every leaf and the release task take `ref`, `branch` (the **logical** branch that drives config/tags/prerelease), and where relevant `smoke`. Branch-derived config keys off `inputs.branch` (the logical branch the caller passes). Artifact names are branch-suffixed. ### Versioning -NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly - no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`; every other branch builds a prerelease `X.Y.Z-g`. `version.json`'s `version` is the major.minor floor; NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code** - they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification; the .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. +NBGV versions the branch being published. Each run builds a single branch (the trigger ref), so `GITHUB_REF` already names it and NBGV classifies it directly - no `IGNORE_GITHUB_REF` override is required. The default branch is the public-release ref, so it builds clean `X.Y.Z`. Every other branch builds a prerelease `X.Y.Z-g`. `version.json`'s `version` is the major.minor floor. NBGV appends the git height as the patch. **NBGV and `version.json` are retained even by a repo with no compiled code** - they are the source of the release tag (`SemVer2`) and `target_commitish` (`GitCommitId`) and the prerelease classification. The .NET SDK is pulled in only as the versioning toolchain. A package build derives its registry version from the same NBGV outputs, but **not always from `SemVer2`**: the PyPI version is built from `AssemblyFileVersion` (four-part `M.N.P.B`) with a PEP 440 `.dev0` appended on the `develop` branch. A wrapper repo may drive its build/image version from an external committed `name -> version` state file while NBGV still tags the release. ### Validate-at-Entry @@ -95,11 +95,11 @@ When a workflow's inputs carry a cross-input or input-versus-derived-state invar ### Resource Lifecycle -Workflow artifacts are an **intra-run handoff** only; durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed - and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. +Workflow artifacts are an **intra-run handoff** only. Durable copies live on the release/registry. The rule: a transfer artifact handed **between jobs** is deleted by exact name/pattern **at its point of consumption**, the delete is **gated to the same condition as the consumer**, and it is **best-effort**. **Every** `upload-artifact` sets `retention-days: 1` as the universal failure-path backstop, so no terminal blanket-delete job is needed - and an intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an aggregation step) may rely on the retention backstop alone. The run is **never** blanket-deleted (`.artifacts[].id`). See D5. ### Fast PR Feedback -PRs validate fast and never publish: a paths-filter smoke-builds only changed targets; a validation job always runs; smoke builds compile/lint/test but upload nothing and push nothing; one required aggregator gates the merge. See D1. +PRs validate fast and never publish: a paths-filter smoke-builds only changed targets. A validation job always runs. Smoke builds compile/lint/test but upload nothing and push nothing. One required aggregator gates the merge. See D1. ```mermaid flowchart TD @@ -114,7 +114,7 @@ flowchart TD ### Release Model -Each publish builds a **single branch** - the trigger ref (`main` a release, `develop` a prerelease) - so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it; a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE; targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish); Docker re-pushes by design. +Each publish builds a **single branch** - the trigger ref (`main` a release, `develop` a prerelease) - so there is no branch matrix and `github.ref` always names the built branch. A **human merge never auto-publishes**: a first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it. A run publishes on a **code-affecting bot push to `main`** (the App merges every Dependabot/codegen PR, so `github.actor` gates it, and a shared paths filter also drops a non-substantive change like an Actions bump), a **manual dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker, to refresh the base image). The `push` is main-only, so a develop bot merge publishes nothing (its prerelease comes via dispatch). A **source-only** repo publishes on **dispatch only**. Every release is a tag on the built commit plus a source archive, README, and LICENSE. Targets amend it with `release-asset-*` files or push to their own registry. An unchanged version re-pushes nothing (no-op republish). Docker re-pushes by design. ```mermaid flowchart TD @@ -130,9 +130,9 @@ flowchart TD Pick each output's path by **where the artifact goes**: - **File on the GitHub release** (zip, binary, packaged library): one leaf per output uploading `release-asset--`. The repo keeps `expect_release_assets: true` (its default). -- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`; PyPI is **split** - the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. -- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image); contributes no `release-asset-*`. -- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited; with the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all - its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). +- **Package-registry push** (NuGet, PyPI): the leaf builds and publishes to its registry. NuGet pushes from the leaf *and* uploads a `release-asset-*`. PyPI is **split** - the leaf only builds + uploads its build artifact, a separate publish job does the OIDC upload (so `id-token: write` is granted at one entry point, behind an environment gate) and contributes **no** `release-asset-*`. +- **Image-registry push** (Docker): the leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only (arm64 emulation is reserved for the released image) - contributes no `release-asset-*`. +- **No file target via the release task** (Docker-only, PyPI-only): the release is tag + source zip + README + LICENSE. The repo's **caller MUST pass `expect_release_assets: false`** to the release task (the input is never set by a publisher that ships file targets, which keeps the default `true`). This is the one case where the otherwise-verbatim publisher is edited. With the default `true` and no assets, the release-create step fails on `fail_on_unmatched_files`. A **source-only** repo has no release task at all - its standalone `publish-release.yml` inlines `action-gh-release`, so `expect_release_assets` does not apply (see Section 6). ## 4. Behavioral Contract - Expected Outcomes @@ -140,17 +140,17 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input ### D1 - PR Fast-Feedback (Smoke) -- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run; unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* -- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test); a non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it - both the aggregator and `smoke-build` (which `needs:` the validation job by name) - to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* +- **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped). *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* +- **D1.2 A validation job always runs.** Input: any PR. Output: a type-appropriate validation job runs unconditionally and the aggregator `needs:` it. In a .NET repo this is the `unit-test` job (format/style/test). A non-.NET repo **replaces** it (not deletes) with its own validator (lint, schema-check) and re-points **every** `needs:` on it - both the aggregator and `smoke-build` (which `needs:` the validation job by name) - to the replacement. *Prevents: a PR merging with no validation, or a dangling `needs:` that fails the whole workflow to load.* - **D1.3 Smoke never publishes and never uploads.** Input: `smoke: true`. Output: full compile/lint/test, but no registry/image push, no release, and **no** artifact uploads (every `upload-artifact`, including any aggregation job, is gated `!smoke`). *Prevents: a PR publishing; orphaned artifacts churning the storage quota.* - **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter excludes workflow files, so smoke-build skips. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* - **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, `needs:` the changes job and the validation job, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* -- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate); `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR - a distinct knob from `fail_ci_if_error` (which only guards the upload step) - and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`; a repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact - `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* +- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --collect:"XPlat Code Coverage"` or `pytest --cov-report=xml`) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** secret store and reaches the reusable validator via `secrets: inherit`. Required for **every** C# and Python repo that has tests (see `spec/secrets.json` `typeMechanisms`). The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR - a distinct knob from `fail_ci_if_error` (which only guards the upload step) - and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact - `.gitignore` excludes it (e.g. `coverage/`, `*.cobertura.xml`; `.gitignore` is the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported; a stale, unused token; a coverage regression blocking an unrelated PR; a coverage artifact committed by a blanket add.* ### D2 - Input/State Validation at Entry -- **D2.1 Validate before expensive work.** Output: a dedicated entry job/step asserts each cross-input/derived-state invariant and fails fast before builds; downstream jobs `needs:` it. -- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none; it strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts); and it is **skipped on smoke** (a detached PR head always versions as prerelease). *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* +- **D2.1 Validate before expensive work.** Output: a dedicated entry job/step asserts each cross-input/derived-state invariant and fails fast before builds. Downstream jobs `needs:` it. +- **D2.2 Release branch matches version classification.** Input: a real (non-smoke) release build. Output: the gate fails loudly if the default branch carries a prerelease suffix **or** a non-default branch carries none. It strips `+buildmetadata` before testing for the prerelease `-` (only a core/prerelease `-` counts), and it is **skipped on smoke** (a detached PR head always versions as prerelease). *Prevents: a non-default leg published as stable; a build-metadata false-positive; the gate blocking every default-base promotion PR.* - **D2.3 Publish only from main or develop.** Input: a dispatch publish. Output: a dispatch from any ref other than `main` or `develop` fails fast. *Prevents: cutting a release from an unintended branch.* - **D2.4 Mutually-exclusive / paired inputs are validated.** Input: a workflow with either/or or must-pair inputs (e.g. the docker-readme task's `repositories` XOR `manifest`+`manifest-jq`). Output: a half-filled or conflicting combination fails fast. *Prevents: a silent fall-through.* @@ -158,28 +158,28 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input - **D3.1 One branch per run.** Input: a publish triggered on `main` or `develop`. Output: the run builds and versions that one branch, and `github.ref` names it, so NBGV classifies it directly (no `IGNORE_GITHUB_REF`). *Prevents: a cross-branch ref mismatch misclassifying the version.* - **D3.2 Default = public, others = prerelease.** Output: default branch -> `X.Y.Z`; any other -> `X.Y.Z-g`. The default-branch literal in the gate, the `prerelease` expression, and `version.json` MUST all name the repo's real default branch. -- **D3.3 Version floor + git height.** Output: `version.json` sets the major.minor floor; NBGV appends the git height as the patch, bumped only for a functional change by the maintainer. NBGV and `version.json` are retained even by a no-compiler repo (they own the tag). -- **D3.4 Registry versions follow the classification, per registry.** Output: NuGet default = stable, others = prerelease (derived by NuGet.org from the SemVer2 `-g` suffix on `PackageVersion`, not a flag the workflow sets). PyPI builds from `AssemblyFileVersion` (`M.N.P.B`) and appends `.dev0` on the `develop` branch only (a two-branch literal, not a generic N-branch rule); the develop `.dev0` build must remain `pip install --pre`-selectable and sort above the default release (NBGV git height in the release segment keeps develop ahead). *Prevents: a non-default leg published as a release; a renamed/extra branch silently getting a plain version.* -- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring - a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`; if the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* +- **D3.3 Version floor + git height.** Output: `version.json` sets the major.minor floor. NBGV appends the git height as the patch, bumped only for a functional change by the maintainer. NBGV and `version.json` are retained even by a no-compiler repo (they own the tag). +- **D3.4 Registry versions follow the classification, per registry.** Output: NuGet default = stable, others = prerelease (derived by NuGet.org from the SemVer2 `-g` suffix on `PackageVersion`, not a flag the workflow sets). PyPI builds from `AssemblyFileVersion` (`M.N.P.B`) and appends `.dev0` on the `develop` branch only (a two-branch literal, not a generic N-branch rule). The develop `.dev0` build must remain `pip install --pre`-selectable and sort above the default release (NBGV git height in the release segment keeps develop ahead). *Prevents: a non-default leg published as a release; a renamed/extra branch silently getting a plain version.* +- **D3.5 Wrapper repos may use an external version.** Output: a repo wrapping an upstream release drives its build/image version from a committed `name -> version` state file, while NBGV still tags the release. *Note: the tracker (the writer) ships without consumer wiring - a wrapper must wire the leaf to read the state file (e.g. `jq` into the image tag) instead of `SemVer2`. If the leaf still tags off NBGV, the wrapper is not actually pinned to upstream.* ### D4 - Release / Publish -- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing; a **human merge never auto-publishes**. A first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it: publish on a **code-affecting bot push to `main`** (gated to the codegen App / Dependabot `github.actor`; an Actions-only bump matches no release path and publishes nothing), a **dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker). A source-only repo publishes on dispatch only. Each run builds one branch. +- **D4.1 Gated single-branch publish.** Output: PRs smoke-test and publish nothing. A **human merge never auto-publishes**. A first `plan` job (`publish-plan-task.yml`) decides once and every job gates on it: publish on a **code-affecting bot push to `main`** (gated to the codegen App / Dependabot `github.actor`; an Actions-only bump matches no release path and publishes nothing), a **dispatch** of `main`/`develop`, or a **main-only weekly schedule** (Docker). A source-only repo publishes on dispatch only. Each run builds one branch. - **D4.2 Tag the built commit.** Output: the release `target_commitish` is the built commit's SHA (NBGV's commit id), never `github.sha` or a moving branch ref. *Prevents: the tag landing on the default branch instead of the built tree.* -- **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download); with the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. -- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed - the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it; registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence - they run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* +- **D4.3 Release contents.** Output: every release is a tag on the built commit plus the auto source zip, README, and LICENSE; file-producing targets attach `release-asset-*`; `prerelease` equals `branch != default`. A no-file-target repo that uses the release task (Docker-only, PyPI-only) reaches the tag-only shape **only** with `expect_release_assets: false` set by the caller (which relaxes `fail_on_unmatched_files` and skips the asset download). With the default `true` and no assets the release-create step fails. A source-only repo reaches the same shape through its inlined `action-gh-release` instead, with no release task or `expect_release_assets`. +- **D4.4 No-op republish.** Input: a re-run whose version is unchanged. Output: nothing is re-pushed - the release-create step is skipped when the tag exists (refreshed only on `workflow_dispatch`), and the paired asset-delete is skipped with it. Registry pushes are no-ops. The NuGet/PyPI publish steps are **not** statically gated on existence - they run and the **server** dedupes (`dotnet nuget push --skip-duplicate` turns a 409 into success; PyPI `skip-existing: true`). **Docker always re-pushes** the image (base-image refresh), independently of the release-create skip, within the same run. *Prevents: duplicate releases and wasted pushes.* ### D5 - Resource Cleanup - **D5.1 Delete at the point of consumption.** Output: the job that downloads a **cross-job** transfer artifact deletes it (by exact name/pattern) right after consuming it. An intermediate consumed only within the same run (e.g. an executable's per-runtime outputs feeding an in-run aggregation) MAY instead rely on the `retention-days: 1` backstop. *Prevents: transfer artifacts accumulating against the storage quota.* -- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too; where the consumer always runs when its job runs (the PyPI publish step), the delete always runs - so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* +- **D5.2 Gate the delete to the consumer's condition.** Output: the delete runs under the **same** condition as its consuming step. Where the consumer is conditional (the GitHub release create), the delete is conditional too. Where the consumer always runs when its job runs (the PyPI publish step), the delete always runs - so on a no-op re-run the `release-asset-*` delete is **skipped** while the PyPI build-artifact delete still **runs** (its publish ran). *Prevents: deleting freshly built assets on a no-op re-run.* - **D5.3 Best-effort.** Output: cleanup is `continue-on-error`, tolerates a failed listing, and deletes **all** matching ids. *Prevents: a cleanup hiccup reddening a job whose publish succeeded.* - **D5.4 Retention backstop.** Output: **every** `upload-artifact` sets `retention-days: 1`. - **D5.5 Never blanket-delete.** Output: cleanup MUST NOT enumerate and delete the run's whole artifact set. *Prevents: destroying diagnostic/log artifacts and auto-emitted build-records.* ### D6 - Seam / Architecture Conformance -- **D6.1 Pattern handoff.** Output: the release job downloads by `pattern:`/`merge-multiple:`, not `artifact-ids:`; targets upload `release-asset--`. Canonical for single-target. +- **D6.1 Pattern handoff.** Output: the release job downloads by `pattern:`/`merge-multiple:`, not `artifact-ids:`. Targets upload `release-asset--`. Canonical for single-target. - **D6.2 Branch drives config.** Output: branch-derived config reads `inputs.branch`, never `github.ref_name`. - **D6.3 Branch-suffixed artifacts.** Output: artifact names are branch-suffixed so a branch's artifacts do not collide with another branch's. - **D6.4 Target add/drop is consistent.** Output: adding or dropping a target updates **all** of: the `enable_` input, the `build-` job and its `github-release` `needs:` entry, the `changes` paths-filter entry + output, and the `smoke-build` enable-forward (and, for PyPI, the separate `publish-pypi` job). The `github-release` job body stays verbatim. *Prevents: a partial subset that startup-fails on a missing leaf or never smoke-builds a target.* @@ -187,22 +187,22 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as input ### D7 - Concurrency, Permissions, Safety - **D7.1 Publisher serializes.** Output: the publisher uses a **global, ref-independent** concurrency group with `cancel-in-progress: false`. *Prevents: a schedule and a dispatch double-pushing, or a cancelled publish leaving a partial release.* -- **D7.2 Skipped jobs still need valid permissions.** Output: every reusable job declares valid `permissions:`; a callee's extra scope (e.g. `actions: write` for cleanup, or `id-token: write` for OIDC) is granted by the caller and appears at exactly the one entry point that needs it. +- **D7.2 Skipped jobs still need valid permissions.** Output: every reusable job declares valid `permissions:`. A callee's extra scope (e.g. `actions: write` for cleanup, or `id-token: write` for OIDC) is granted by the caller and appears at exactly the one entry point that needs it. - **D7.3 Boolean inputs both forms.** Output: declared in both trigger blocks, compared against `true` and `'true'`. - **D7.4 Optional-dependency chaining.** Output: cross-job conditions allowlist `success`/`skipped` explicitly. ### D8 - Bots / Automation - **D8.1 Merge-bot.** Output: enables auto-merge on `opened`/`reopened` for **every** Dependabot tier including semver-major (the required checks are the gate, not the bump magnitude); dispatches `--squash`/`--merge` by the PR's base ref; disables on a maintainer-pushed `synchronize`; concurrency keyed on the **PR number**, not `github.ref`. *Prevents: two PRs colliding in auto-merge.* -- **D8.2 CodeGen and Dependabot.** Output: codegen runs as a matrix over both branches and is deterministic from an external source; Dependabot targets both branches, security PRs to default. -- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it; the `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish - it ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `-` head/base pairs, or auto-merge silently never fires. +- **D8.2 CodeGen and Dependabot.** Output: codegen runs as a matrix over both branches and is deterministic from an external source. Dependabot targets both branches, security PRs to default. +- **D8.3 Upstream-version tracker.** Output: a scheduled resolver prints a JSON `name -> version` object to a committed state file, opens a rolling per-branch bump PR naming only the moved keys, the merge-bot auto-merges it. The `main` pin push publishes via the release gate, while a `develop` pin does not auto-publish - it ships via a `develop` dispatch (prerelease) or the next promotion to `main`. The tracker's `bump-branch-prefix` + `branches` MUST match the merge-bot's hard-coded `-` head/base pairs, or auto-merge silently never fires. - **D8.4 An identity allowlist used as a gate fails loud.** Where a gate compares `github.actor` (or a PR author) against hard-coded bot identities, the non-matching branch on an otherwise-legitimate trigger **emits a `::warning::`** rather than falling through silently. Output: a run that declines to act on an unrecognized identity is visibly annotated. *Prevents: the App being renamed, replaced, or reinstalled under a new slug, after which the comparison quietly evaluates false and the gate stops firing - a green, silent run that looks identical to a healthy one.* The masking matters most where a second path hides the loss: a weekly schedule keeps publishing, so the only symptom is release *timeliness*, easily missed for months. Where the failure is self-announcing instead (the merge-bot simply stops merging, so bot PRs visibly pile up) an annotation is optional. Resolving the identity at run time (mint an App token, read `GET /app`) removes the hard-coded string entirely and is the escalation if an allowlist proves fragile in practice. ### D9 - Style / Static (See Section 2) - **D9.1** Every action SHA-pinned with a version comment (sole exception: the documented lagging-tag tool). -- **D9.2** File/workflow/job/step names follow the suffix rules; a ruleset-bound job's `name:` equals its ruleset `context:` (renamed together). -- **D9.3** Bash `run:` blocks start `set -Eeuo pipefail`; multi-line `if:` uses `>-`. +- **D9.2** File/workflow/job/step names follow the suffix rules. A ruleset-bound job's `name:` equals its ruleset `context:` (renamed together). +- **D9.3** Bash `run:` blocks start `set -Eeuo pipefail`. Multi-line `if:` uses `>-`. - **D9.4** Docker layer cache targets a registry tag, not `type=gha`; `cache-to` writes only the built branch's `buildcache-` and only on push, while `cache-from` reads both branches; multi-image repos use a per-image cache tag. - **D9.5** Line endings follow `.editorconfig`. @@ -219,18 +219,18 @@ Read the workflow files plus `version.json` and assert the structural fact behin - **D1:** a `changes` paths-filter job exists, covers each of the repo's targets, and **excludes** `.github/workflows/**`; the PR entry workflow's smoke call sets `github/nuget/dockerhub: false` on the release task; the leaf receives `smoke: true` and a derived `push` (false on smoke); every build-task `upload-artifact` (and any aggregation job) is gated `!smoke`; the aggregator `needs:` the `changes` and validation jobs, blocks on `failure`/`cancelled`, passes on `skipped`; a validation job runs unconditionally. - **D2:** an entry validation job/step exists per complex-input workflow; the release gate checks both directions, strips `+buildmetadata`, and skips on smoke; the publisher rejects a dispatch from a ref other than `main` or `develop`. - **D3:** each run builds one branch, so NBGV classifies `github.ref` directly (no `IGNORE_GITHUB_REF`); the default-branch literal in the gate (`== 'main'`), the `prerelease` expression (`!= 'main'`), and `version.json`'s `publicReleaseRefSpec` all name the repo's actual default branch. -- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely - every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional; record the gate N/A there, not missing. +- **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step is gated identically. A dispatch-only publisher (`releaseTrigger: dispatch-only`) may omit the gate and the exists-check entirely - every run is a dispatch, so the skip leg can never fire and create-or-refresh is unconditional. Record the gate N/A there, not missing. - **D5:** each cross-job transfer artifact has a delete step at its consumer, gated to the consumer's condition, `continue-on-error: true`, looping all ids; **every** upload sets `retention-days: 1`; **no** `.artifacts[].id` blanket delete exists anywhere. -- **D6:** the release download uses `pattern:`/`merge-multiple:` (no `artifact-ids:`); branch-derived config reads `inputs.branch` (a `github.ref_name` in such config is a finding); artifact names are branch-suffixed; the target set is consistent across the release task and the paths-filter. -- **D7:** the publisher concurrency group is ref-independent with `cancel-in-progress: false`; reusable jobs declare permissions; boolean `if:` uses both forms. -- **D8/D9:** merge-bot concurrency keys on PR number; the upstream tracker's branch prefix matches the merge-bot's head-ref pairs (wrapper repos); actions are SHA-pinned; names/shells/conditionals follow section 2. +- **D6:** the release download uses `pattern:`/`merge-multiple:` (no `artifact-ids:`). Branch-derived config reads `inputs.branch` (a `github.ref_name` in such config is a finding). Artifact names are branch-suffixed. The target set is consistent across the release task and the paths-filter. +- **D7:** the publisher concurrency group is ref-independent with `cancel-in-progress: false`. Reusable jobs declare permissions. Boolean `if:` uses both forms. +- **D8/D9:** merge-bot concurrency keys on PR number. The upstream tracker's branch prefix matches the merge-bot's head-ref pairs (wrapper repos). Actions are SHA-pinned. Names/shells/conditionals follow section 2. **Per-type addenda (apply only the ones present):** -- **Console/executable:** the smoke runtime matrix is a strict non-empty subset of the full matrix; the per-runtime outputs (`publish--`) are aggregated by `pattern:` + `merge-multiple:` into one `release-asset--` and the aggregation job is gated `!smoke`; the per-runtime intermediates rely on the retention backstop (no explicit delete is required for an in-run intermediate). -- **NuGet:** the publish step is gated `if: inputs.push` only (not on an existence check) and uses `--skip-duplicate`; `*.nupkg` push also carries the paired `.snupkg` to the symbol server where symbols are enabled; the `release-asset` zip carries the package(s). -- **PyPI:** `publish-pypi` declares `environment: { name: pypi }`; `id-token: write` appears only on that job (absent from the build/PR path); `skip-existing: true` is set on the publish action; the build artifact is deleted after publish; the `pypi` environment has a deployment-branch rule. -- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`; the leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only - a plain Docker repo correctly tags off `SemVer2` and records this N/A); the readme/date-badge jobs are gated main-only; the docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`; the buildcache follows D9.4. +- **Console/executable:** the smoke runtime matrix is a strict non-empty subset of the full matrix. The per-runtime outputs (`publish--`) are aggregated by `pattern:` + `merge-multiple:` into one `release-asset--` and the aggregation job is gated `!smoke`. The per-runtime intermediates rely on the retention backstop (no explicit delete is required for an in-run intermediate). +- **NuGet:** the publish step is gated `if: inputs.push` only (not on an existence check) and uses `--skip-duplicate`. `*.nupkg` push also carries the paired `.snupkg` to the symbol server where symbols are enabled. The `release-asset` zip carries the package(s). +- **PyPI:** `publish-pypi` declares `environment: { name: pypi }`. `id-token: write` appears only on that job (absent from the build/PR path). `skip-existing: true` is set on the publish action. The build artifact is deleted after publish. The `pypi` environment has a deployment-branch rule. +- **Docker:** a Docker-only repo's caller passes `expect_release_assets: false`. The leaf reads the external state file for the tag instead of `SemVer2` (wrapper repos only - a plain Docker repo correctly tags off `SemVer2` and records this N/A). The readme/date-badge jobs are gated main-only. The docker-readme task validates `repositories` XOR `manifest`+`manifest-jq`. The buildcache follows D9.4. ### 5B. End-to-End Trace Scenarios (No Execution, Deterministic from the YAML) @@ -254,27 +254,27 @@ For each *applicable* scenario, evaluate every job's `if:`/`needs:` against the - Open a trivial-change PR touching one target and confirm S1. - Drive a `smoke: true` push-probe of the build task for **both** the default and a non-default branch and assert the version classification (clean vs prerelease) and that the gate passes - **without publishing**. *Caveat: the Docker leg logs in to the registry even on smoke and reads the buildcache, so it needs `DOCKER_HUB_*` secrets and cannot run on a fork PR (same-repo only).* -- Per registry: after a real publish, query NuGet.org for the expected version + prerelease classification (and the `.snupkg` on the symbol server), and confirm a re-run added no duplicate; for PyPI inspect the `Compute PyPI version step` log and the built `dist/*` filenames for `.dev0` off `develop` vs a plain version on the default branch. +- Per registry: after a real publish, query NuGet.org for the expected version + prerelease classification (and the `.snupkg` on the symbol server), and confirm a re-run added no duplicate. For PyPI inspect the `Compute PyPI version step` log and the built `dist/*` filenames for `.dev0` off `develop` vs a plain version on the default branch. - Inspect the latest real publish's logs for `PublicRelease`/`SemVer2` per leg and confirm the artifact lifecycle (uploaded, consumed, deleted; none left behind). ### Assessment The workflow is **operational** iff every *applicable* 5A item passes and every *applicable* 5B scenario's observed output equals the expected (confirmed by 5C where a live signal exists). N/A items are excluded, never counted as failures. Any *applicable* mismatch is a **defect** -> **not operational**. Procedure: -1. **Audit** with 5A; record pass/fail/N-A with `file:line`. -2. **Trace** the applicable S-scenarios with 5B; diff predicted vs expected. +1. **Audit** with 5A. Record pass/fail/N-A with `file:line`. +2. **Trace** the applicable S-scenarios with 5B. Diff predicted vs expected. 3. **Probe** with 5C only for guarantees a static trace cannot settle (live version classification, registry state, artifact lifecycle). 4. **Verdict:** operational / not operational, with the failing guarantee(s) and the triggering input for each, and the list of items recorded N/A. ## 6. Per-Project-Type Test Walkthroughs -Each type maps the *applicable* S-scenarios onto its targets; the differences are which leaf tasks exist and what each produces, which 5A addenda apply, and which scenarios are N/A. Walking these is the self-check that the contract holds for each shape. +Each type maps the *applicable* S-scenarios onto its targets. The differences are which leaf tasks exist and what each produces, which 5A addenda apply, and which scenarios are N/A. Walking these is the self-check that the contract holds for each shape. -- **Console / executable application.** Target produces `release-asset--executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish--` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes; the per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest"; the workflow does not set it). -- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset--nugetlibrary`; configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it - a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. -- **PyPI library.** The leaf builds + uploads `pypilibrary-build-`; a **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact - **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`; a PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. -- **Docker image.** The leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only, with a per-branch registry buildcache (`buildcache-`; a multi-image repo adds a per-image tag) (`cache-to` only the built branch and only on push, `cache-from` both branches); no `release-asset-*`, so a Docker-only repo's caller passes `expect_release_assets: false`; the readme (`peter-evans/dockerhub-description`, `DOCKER_HUB_ACCESS_TOKEN`) and date-badge jobs run **only** when the default branch publishes; the docker-readme task validates `repositories` XOR `manifest`+`manifest-jq` and a multi-image repo derives its publish matrix from the manifest. Docker **always re-pushes** the image, independently of a skipped release-create (S9). A **wrapper** repo tracks an upstream release: the upstream tracker writes a `name -> version` state file and the merge-bot auto-merges the bump PR (S11), and the leaf MUST read that file for the immutable tag instead of `SemVer2` (the tracker ships without this consumer wiring). Test: S7 default leg pushes `latest` + the version tag and updates readme/badge; non-default pushes the develop tag (amd64 only); S9 still re-pushes; S11 ships the bumped upstream version next publish. 5C Docker probe needs `DOCKER_HUB_*` secrets and same-repo (not fork) runs. -- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset--library` (`retention-days: 1`, upload gated `!smoke` - mirror the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5); `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. +- **Console / executable application.** Target produces `release-asset--executable` (a 7z archive, `Console.7z`) by building a per-runtime `dotnet publish` matrix, then an aggregation job downloads the per-runtime `publish--` intermediates (`pattern:` + `merge-multiple:`), zips them, and uploads the single asset. Smoke builds a strict subset of runtimes. The per-runtime upload **and** the aggregation job are both gated `!smoke`, so smoke uploads nothing. The per-runtime intermediates rely on `retention-days: 1` (no explicit delete). Test: S1 with a console change smoke-builds the subset and uploads nothing; S7 attaches the 7z, `prerelease=true` on the non-default leg and `prerelease=false` on the default leg (GitHub auto-marks the stable default release "Latest" - the workflow does not set it). +- **NuGet library.** The leaf both pushes (`dotnet nuget push *.nupkg --skip-duplicate`, gated `if: push` only) and uploads `release-asset--nugetlibrary`. Configuration is Release on the default branch, Debug otherwise. Where symbols are enabled (`snupkg`), the push auto-carries the paired `.snupkg` to NuGet.org's symbol server and the asset zip also contains it - a triple surface. NuGet.org derives `isPrerelease` from the SemVer2 `-g` suffix (the workflow sets no such flag). Test: S7 non-default leg publishes a prerelease package + asset, default a stable; S9 re-run is a server-side `--skip-duplicate` no-op. 5C: query NuGet.org for both versions and the symbol package. +- **PyPI library.** The leaf builds + uploads `pypilibrary-build-`. A **separate** `publish-pypi` job (with `environment: pypi`, `id-token: write`, `actions: write`) does the OIDC Trusted-Publishing upload with `skip-existing: true`, then **consume-then-deletes** the build artifact - **unconditionally on consume**, so on S9 it is deleted even though the `release-asset-*` delete is skipped. The version is `AssemblyFileVersion` with `.dev0` appended on `develop` only, and must stay `--pre`-selectable and sorted above the default release. PyPI contributes no `release-asset-*`. A PyPI-only repo sets `expect_release_assets: false` at the caller. Test: S7 default leg publishes a release, non-default a `.dev0`; S9 is a `skip-existing` no-op; 5C inspects the `dist/*` filenames and the compute-version log. +- **Docker image.** The leaf pushes the default branch multi-arch (amd64+arm64) and any other branch `amd64`-only, with a per-branch registry buildcache (`buildcache-`; a multi-image repo adds a per-image tag) (`cache-to` only the built branch and only on push, `cache-from` both branches); no `release-asset-*`, so a Docker-only repo's caller passes `expect_release_assets: false`; the readme (`peter-evans/dockerhub-description`, `DOCKER_HUB_ACCESS_TOKEN`) and date-badge jobs run **only** when the default branch publishes; the docker-readme task validates `repositories` XOR `manifest`+`manifest-jq` and a multi-image repo derives its publish matrix from the manifest. Docker **always re-pushes** the image, independently of a skipped release-create (S9). A **wrapper** repo tracks an upstream release: the upstream tracker writes a `name -> version` state file and the merge-bot auto-merges the bump PR (S11), and the leaf MUST read that file for the immutable tag instead of `SemVer2` (the tracker ships without this consumer wiring). Test: S7 default leg pushes `latest` + the version tag and updates readme/badge. Non-default pushes the develop tag (amd64 only). S9 still re-pushes. S11 ships the bumped upstream version next publish. 5C Docker probe needs `DOCKER_HUB_*` secrets and same-repo (not fork) runs. +- **Data / asset library.** A single new leaf: validate -> zip -> upload `release-asset--library` (`retention-days: 1`, upload gated `!smoke` - mirror the nugetlibrary leaf's shape). Because no such leaf ships, you **add a target** (D6.4): a new `enable_library` input + `build-library` job + `github-release` `needs:` entry in the release task, and a `library` paths-filter entry + `changes` output + `smoke-build` enable-forward in the PR workflow (without it, D1.1 never smoke-builds the library). Keep `expect_release_assets: true` (it has a file target, unlike Docker). The .NET `unit-test` job is replaced by a type-appropriate validator with the aggregator **and** `smoke-build` both re-pointed to it (D1.2/D1.5). `version.json` + the NBGV `get-version` step are retained (they own the tag). Test: S1 smoke runs validate+zip and uploads nothing; S7 attaches the zip, prerelease on the non-default leg; S9 on a *scheduled* re-run release-create + asset-delete skip (the existing zip is untouched, no registry push), while a `workflow_dispatch` re-run **refreshes** the release and re-runs the asset-delete (the asset is re-uploaded then re-deleted). N/A: the nuget/pypi/docker/executable 5A addenda and their scenario clauses. - **Source-only / no build.** There is no `build-release-task.yml` (its `appliesTo` excludes source-only) and no package/image leaf, so nothing is edited down. The release is a standalone dispatch-only `publish-release.yml` that inlines NBGV for the tag and `action-gh-release` for the release - tag + source zip + README + LICENSE, no reusable release task and no asset download. With no target the paths-filter matches nothing, so `smoke-build` is **structurally always skipped** - validation is carried solely by the (replaced, non-.NET) validation job that the aggregator and `smoke-build`'s own `needs:` must both point at (D1.2; or drop the never-running `smoke-build` job). NBGV and `version.json` are still retained (they own the tag). Its publish job gates on the repo's reusable validation task (`needs:` the same `workflow_call` job the PR workflow runs), so a dispatch cannot release a ref that fails validation. Applicable scenarios: S1 (validation only), S5/S6 (publish gating), S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification gate). N/A: S2-S4 (assume a smoke-built target), the artifact-lifecycle and registry clauses of S7/S9, the D5/D6 artifact items, and all per-type 5A addenda - recorded N/A, not failed. - **Operational (workflow model, not a build target).** A `workflowModel: operational` repo layers a direct-commit `develop` onto the **source-only** release shape (above). Two workflows: (1) a **lint/validation** PR workflow feeding the required `Check pull request workflow status job` - the generic linters (editorconfig/EOL, markdownlint, cspell, actionlint) plus a domain validator (Home Assistant `hass --script check_config`, `esphome config`, a firmware build), **no unit tests**; its triggers differ from the `release` model - `push` to `develop` (advisory feedback on direct commits) plus `pull_request` to `main` (the enforced promotion gate) plus `workflow_dispatch`. (2) the standard **source-only publisher** on `workflow_dispatch` only (`releaseTrigger: dispatch-only`): NBGV + `version.json` own the tag, and a manual dispatch cuts a GitHub release (tag + source zip + README + LICENSE, via the standalone publisher's inlined `action-gh-release`). Applicable scenarios: S1 (validation) on the promotion PR, plus the source-only release set - S7 (tag-only release), S8 (dispatch guard), S9 (no-op republish), S10 (classification). N/A: the auto-publish paths (S5/S6 bot-push and schedule - operational repos have neither) and every build/registry scenario. See the branch-model note in Section 3 and [AGENTS.md "Branching Model"][agents-branching-model]. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..08780d8c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,18 @@ +# Config only (no [project]/[build-system]/uv.lock) - the Scripts profile, CODESTYLE.md "Two profiles". + +[tool.ruff] +target-version = "py313" +line-length = 100 + +[tool.ruff.lint] +extend-select = ["I"] # isort import ordering, on top of the default rules + +[tool.mypy] +python_version = "3.13" +files = ["spec", "host-setup"] + +[tool.pyright] +pythonVersion = "3.13" +typeCheckingMode = "standard" +include = ["spec", "host-setup"] +exclude = ["**/__pycache__"] diff --git a/reports/divergences.md b/reports/divergences.md new file mode 100644 index 00000000..fa28eaf4 --- /dev/null +++ b/reports/divergences.md @@ -0,0 +1,42 @@ +# Fleet divergence report + +Generated by `python3 spec/fidelity_honesty.py --report` - do not hand-edit. Curate dispositions in [`spec/divergences.json`][ledger] and regenerate. Each row reflects a repo's ground-truth branch at generation time. Git dates this file. + +## Burn-down + +### re-vendor + +- **.markdownlint-cli2.jsonc** - AudioCleaner, PhotoCleaner, aiopurpleair - Verbatim config held as a hand-modified copy in these three (not a past hub revision). Restore the current canonical. + +### upstream-candidate + +- **repo-config/configure.sh** - NxWitness - Not stale - a forked design: repo-specialized (Docker Hub image list, secret names, Make/Matrix.json product matrix) and adds a check/5D-audit mode the hub canonical lacks. Overwriting would regress it. Reconcile by adopting the check mode into the hub canonical, then re-vendoring. + +### investigate + +- **repo-config/configure.sh** - ESPHome-NonRoot, LanguageTags, VSCode-Server-DotNetCore, aiopurpleair, homeassistant-purpleair - Diverge from the 119-line apply-only canonical in a non-stale way, likely carrying the older check-mode design (same family as the NxWitness fork). Triage per repo: fold into the upstream-candidate decision, or re-vendor if merely a forked-then-abandoned copy. +- **pyproject.toml** (manifest gap) - The hub gained a config-only Scripts-profile pyproject.toml in #388. Decide whether to track it (intent, appliesTo python) after confirming the python repos carry an equivalent. +- **.github/workflows/publish-release.yml** (manifest gap) - Carried by some repos, absent from others, and varies widely (12 divergent, 5 absent). Needs a fidelity call (interface vs intent) and an appliesTo scope before tracking - it would surface many new findings. +- **.github/workflows/validate-task.yml** (manifest gap) - As publish-release.yml (11 divergent, 9 absent): fidelity plus appliesTo decision pending. + +### accepted + +- **.editorconfig-checker.json** - HolidayLights, HomeAutomation-Config - Both carry a legitimate repo-specific Exclude list (HomeAutomation-Config excludes a Vantage/ subtree, HolidayLights excludes .fseq sequence files). The uniform Disable block is carried intent-equivalent. Exclude is inherently repo-local, which is why the unit is intent, not verbatim. +- **LICENSE** (manifest gap) - Each repo owns its license file. The hub does not standardize license text, so it is intentionally outside the manifest. + +## Untriaged - add a disposition to `spec/divergences.json` + +_None - every live divergence has a recorded disposition._ + +## Mechanical re-vendor (verbatim stale copies) + +A past hub revision, not the current canonical - the audit already flags these as DRIFT. Copy the current file down. No judgment needed. + +- **.markdownlint-cli2.jsonc** (15): DevKitCIoT, ESPHome-Config, ESPHome-NonRoot, HolidayLights, HomeAssistant-Config, HomeAutomation-Config, KiCadLibrary, LanguageTags, MediaTools, NxWitness, PlexCleaner, Utilities, VSCode-Server-DotNetCore, Vantage-Config, homeassistant-purpleair +- **repo-config/configure.sh** (6): ESPHome-Config, HomeAssistant-Config, HomeAutomation-Config, PlexCleaner, Utilities, Vantage-Config + +## Promote candidates (intent uniform -> verbatim) + +_None - no intent unit is currently fleet-uniform with the canonical._ + +[ledger]: ../spec/divergences.json diff --git a/spec/audit.py b/spec/audit.py index 13ea85b8..c6aa520c 100644 --- a/spec/audit.py +++ b/spec/audit.py @@ -26,6 +26,7 @@ import subprocess import sys from datetime import datetime, timezone +from typing import Any ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -59,8 +60,8 @@ def hub_name(): HUB_NAME, HUB_NAME_FROM_REMOTE = hub_name() -def gh(path, ok404=False): - """GET a REST path via gh, returning parsed JSON or None on 404 when ok404. +def gh(path, ok404=False) -> Any: + """GET a REST path via gh, returning parsed JSON, or None on a 404 (when ok404) or an empty response body. No --paginate: on object endpoints it concatenates page documents into unparseable JSON. Every list read here fits one page; callers pass per_page=100 where a default page could truncate. @@ -276,7 +277,7 @@ def classify_verbatim(down_text, canon_text, past_texts): return "modified" -_HISTORY_CACHE = {} # rel_path -> [past revision content], reused as a canonical is compared against every audited repo +_HISTORY_CACHE: dict[str, list[str]] = {} # rel_path -> past revision contents, cached because one canonical is compared against every audited repo def git_file_history(rel_path): @@ -506,7 +507,7 @@ def audit_repo(entry, spec): # to ""), whereas a too-large or non-inline payload returns encoding "none" (text stays None -> flagged). text = base64.b64decode(content["content"]).decode("utf-8", "replace") if content.get("encoding") == "base64" else None # Interface conformance (name + wiring) plus any verbatim job regions the contract pins. - if fid == "interface": + if item is not None and fid == "interface": if text is None: findings.append(("DRIFT", f"interface: could not read {path} content on {ground} to verify its contract (no inline content returned); verify by hand")) else: @@ -517,7 +518,7 @@ def audit_repo(entry, spec): findings.extend(check_verbatim(f"{path} job '{job}'", text, canonical_rel, extract=lambda t, j=job: split_jobs(t).get(j))) # Whole-file verbatim: byte-identical to the hub's canonical after EOL normalization. - elif fid == "verbatim": + elif item is not None and fid == "verbatim": if text is None: findings.append(("DRIFT", f"verbatim: could not read {path} content on {ground} to compare (no inline content returned); verify by hand")) else: diff --git a/spec/divergences.json b/spec/divergences.json new file mode 100644 index 00000000..c9388809 --- /dev/null +++ b/spec/divergences.json @@ -0,0 +1,16 @@ +{ + "$schema": "./divergences.schema.json", + "note": "Curated dispositions for known fleet divergences from the manifest canonicals - the burn-down ledger. spec/fidelity_honesty.py --report joins this against live fleet reality to write reports/divergences.md. A recorded divergence still present renders as a burn-down task with its disposition. A live divergence absent here renders as UNTRIAGED. A recorded divergence no longer live renders as resolved. Edit this file (not the generated report) and regenerate. dispositions cover per-repo file divergences from a verbatim or intent canonical. gaps cover files carried by the fleet but absent from spec/files.json. disposition vocabulary: re-vendor (drift-to-fix, copy the current canonical down), track (a gap to add to the manifest), accepted (a legitimate permanent divergence, no action), upstream-candidate (the downstream carries an improvement the hub should adopt, then re-vendor), investigate (recorded, decision pending).", + "dispositions": [ + { "path": ".editorconfig-checker.json", "repos": ["HomeAutomation-Config", "HolidayLights"], "disposition": "accepted", "reason": "Both carry a legitimate repo-specific Exclude list (HomeAutomation-Config excludes a Vantage/ subtree, HolidayLights excludes .fseq sequence files). The uniform Disable block is carried intent-equivalent. Exclude is inherently repo-local, which is why the unit is intent, not verbatim.", "tracking": null }, + { "path": ".markdownlint-cli2.jsonc", "repos": ["aiopurpleair", "PhotoCleaner", "AudioCleaner"], "disposition": "re-vendor", "reason": "Verbatim config held as a hand-modified copy in these three (not a past hub revision). Restore the current canonical.", "tracking": null }, + { "path": "repo-config/configure.sh", "repos": ["NxWitness"], "disposition": "upstream-candidate", "reason": "Not stale - a forked design: repo-specialized (Docker Hub image list, secret names, Make/Matrix.json product matrix) and adds a check/5D-audit mode the hub canonical lacks. Overwriting would regress it. Reconcile by adopting the check mode into the hub canonical, then re-vendoring.", "tracking": null }, + { "path": "repo-config/configure.sh", "repos": ["LanguageTags", "aiopurpleair", "homeassistant-purpleair", "ESPHome-NonRoot", "VSCode-Server-DotNetCore"], "disposition": "investigate", "reason": "Diverge from the 119-line apply-only canonical in a non-stale way, likely carrying the older check-mode design (same family as the NxWitness fork). Triage per repo: fold into the upstream-candidate decision, or re-vendor if merely a forked-then-abandoned copy.", "tracking": null } + ], + "gaps": [ + { "path": "LICENSE", "disposition": "accepted", "reason": "Each repo owns its license file. The hub does not standardize license text, so it is intentionally outside the manifest.", "tracking": null }, + { "path": "pyproject.toml", "disposition": "investigate", "reason": "The hub gained a config-only Scripts-profile pyproject.toml in #388. Decide whether to track it (intent, appliesTo python) after confirming the python repos carry an equivalent.", "tracking": null }, + { "path": ".github/workflows/publish-release.yml", "disposition": "investigate", "reason": "Carried by some repos, absent from others, and varies widely (12 divergent, 5 absent). Needs a fidelity call (interface vs intent) and an appliesTo scope before tracking - it would surface many new findings.", "tracking": null }, + { "path": ".github/workflows/validate-task.yml", "disposition": "investigate", "reason": "As publish-release.yml (11 divergent, 9 absent): fidelity plus appliesTo decision pending.", "tracking": null } + ] +} diff --git a/spec/divergences.schema.json b/spec/divergences.schema.json new file mode 100644 index 00000000..27d8989b --- /dev/null +++ b/spec/divergences.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ptr727/ProjectTemplate/spec/divergences.schema.json", + "title": "Fleet divergence dispositions (burn-down ledger)", + "type": "object", + "additionalProperties": false, + "properties": { + "$schema": { "type": "string" }, + "note": { "type": "string" }, + "dispositions": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "repos", "disposition", "reason"], + "additionalProperties": false, + "properties": { + "path": { "type": "string" }, + "repos": { "type": "array", "items": { "type": "string" }, "minItems": 1 }, + "disposition": { "enum": ["re-vendor", "track", "accepted", "upstream-candidate", "investigate"] }, + "reason": { "type": "string" }, + "tracking": { "type": ["string", "null"] } + } + } + }, + "gaps": { + "type": "array", + "items": { + "type": "object", + "required": ["path", "disposition", "reason"], + "additionalProperties": false, + "properties": { + "path": { "type": "string" }, + "disposition": { "enum": ["re-vendor", "track", "accepted", "upstream-candidate", "investigate"] }, + "reason": { "type": "string" }, + "tracking": { "type": ["string", "null"] } + } + } + } + } +} diff --git a/spec/fidelity_honesty.py b/spec/fidelity_honesty.py new file mode 100644 index 00000000..e2ccc9bb --- /dev/null +++ b/spec/fidelity_honesty.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Fidelity-honesty analysis: check the manifest's declared fidelities against fleet reality. + +Read-only, owner-run, not wired into CI. Reuses spec/audit.py's fleet machinery (gh, content_hash, +git history, selectors) - import-safe because audit.py guards its main. + +The verbatim engine verifies a unit's *content* against the canonical (declared -> hashed). This tool +verifies the *classifications themselves* (declared -> checked), the same declared-to-verified leap one +level up. It answers two questions the audit cannot: + + 1. Which `intent` units are actually content-identical (after EOL normalization) across the whole fleet? Those are candidates to + promote to `verbatim` - they would gain free drift-detection (a stale-but-present copy is invisible + under intent, caught under verbatim). This is the class that hid the configure.sh drift. + 2. Which `verbatim` units have a downstream copy that diverges in a NON-stale way? That is either a + mis-set label (the content legitimately varies -> should be intent) or real drift to chase. + +It also runs a manifest-gap pass: a file present in BOTH the hub and a reference adopter but absent from +the manifest is carried-but-untracked (exactly the configure.sh / settings.json bug). + +Usage: python3 spec/fidelity_honesty.py [reference-repo-for-manifest-gap] (default: Financial-Modeling) +""" +import base64 +import subprocess +import sys + +import audit # sibling, import-safe (its main is guarded) + +REF_ADOPTER = "Financial-Modeling" # a well-adopted repo, used only for the manifest-gap pass +REPORT_PATH = "reports/divergences.md" # the generated, checked-in burn-down report (--report) + + +def canonical_text(entry): + """The hub's canonical for a unit: its reference snippet, else its own root copy.""" + ref = entry.get("reference") or entry["path"] + try: + return (audit.ROOT / ref).read_text(encoding="utf-8", errors="replace") + except OSError: + return None + + +def fetch(slug, path, ref): + """Decoded downstream file content, or None if absent / not inline.""" + content = audit.gh(f"repos/{slug}/contents/{path}?ref={ref}", ok404=True) + if content is None or content.get("encoding") != "base64": + return None + return base64.b64decode(content["content"]).decode("utf-8", "replace") + + +def fidelity_pass(spec): + defaults = spec["registry"].get("defaults", {}) + repos = [r for r in spec["registry"]["repos"] if r.get("status") == "cataloged"] + units = [e for e in spec["files"]["baseline"] if e.get("fidelity") in ("intent", "verbatim")] + + spreads = [] # (entry, spread dict) for the full table + promote = [] # intent units that are uniform fleet-wide + mislabel = [] # verbatim units that diverge non-stale + + for e in units: + path, fid = e["path"], e["fidelity"] + canon = canonical_text(e) + if canon is None: + spreads.append((e, None)) + continue + canon_hash = audit.content_hash(canon) + history = {audit.content_hash(t) for t in audit.git_file_history(e.get("reference") or path)} + spread = {"match": [], "stale": [], "differs": [], "unavailable": []} + for r in repos: + if not audit.applies(e.get("appliesTo", "*"), audit.repo_selectors(r, defaults)): + continue + text = fetch(audit.repo_slug(r), path, r.get("groundTruthBranch", "main")) + if text is None: # missing (404) or present-but-non-inline (too large / encoding "none") + spread["unavailable"].append(r["name"]) + continue + dh = audit.content_hash(text) + if dh == canon_hash: + spread["match"].append(r["name"]) + elif dh in history: + spread["stale"].append(r["name"]) + else: + spread["differs"].append(r["name"]) + spreads.append((e, spread)) + # A verbatim candidate has NO hand-modified copy ("differs") and at least one confirmed match with + # the current canonical. Stale copies do not disqualify it - verbatim would flag them "stale -> + # re-vendor", which is the point. A unit that is entirely stale/unavailable is not confirmed uniform. + if fid == "intent" and spread["match"] and not spread["differs"]: + promote.append((e, spread)) + if fid == "verbatim" and spread["differs"]: + mislabel.append((e, spread)) + return spreads, promote, mislabel + + +def manifest_gap_pass(spec, ref_repo): + """Files present in BOTH the hub and the reference adopter but absent from the manifest.""" + listed = {e["path"] for e in spec["files"]["baseline"]} + entry = next((r for r in spec["registry"]["repos"] if r["name"] == ref_repo), None) + if entry is None: + return None, [] + slug = audit.repo_slug(entry) + ground = entry.get("groundTruthBranch", "main") + # The git/trees endpoint takes a tree SHA, not a ref name, so resolve the branch to its tree SHA + # first (as audit.py does) - passing the branch name can 404 and silently drop the whole check. + # Fail loud on an unreadable reference adopter: an empty gaps list would report "none" (a false clean). + br = audit.gh(f"repos/{slug}/branches/{ground}", ok404=True) + if not br or "commit" not in br: + raise RuntimeError(f"could not read {slug}@{ground} (missing branch?) - cannot run the manifest-gap pass") + tree = audit.gh(f"repos/{slug}/git/trees/{br['commit']['commit']['tree']['sha']}?recursive=1", ok404=True) + if not tree or "tree" not in tree: + raise RuntimeError(f"could not read the tree for {slug}@{ground} - cannot run the manifest-gap pass") + # The hub's tracked files (git ls-files), not a filesystem walk: a walk pulls in untracked local cruft + # (__pycache__, a local .venv) and would make the gap report depend on working-tree state. + r = subprocess.run(["git", "ls-files"], cwd=audit.ROOT, capture_output=True, text=True) + if r.returncode != 0: # fail loud: an empty set would masquerade as "no gaps" (a false clean) + raise RuntimeError(f"git ls-files failed in {audit.ROOT}: {r.stderr.strip() or 'non-zero exit'}") + hub_files = set(r.stdout.splitlines()) + gaps = sorted(n["path"] for n in tree["tree"] + if n.get("type") == "blob" and n["path"] in hub_files and n["path"] not in listed) + return slug, gaps + + +def load_ledger(): + """The curated disposition ledger, empty if absent (each verbatim divergence and manifest gap then reads as untriaged). + + Normalized so a malformed file (non-object root, or a non-array dispositions/gaps) degrades to an empty + section rather than crashing render_report. validate.py reports the malformation loudly in CI. + """ + try: + led = audit.load("spec/divergences.json") + except FileNotFoundError: + led = {} + if not isinstance(led, dict): + led = {} + for key in ("dispositions", "gaps"): + if not isinstance(led.get(key), list): + led[key] = [] + return led + + +def _fmt(repos): + return ", ".join(sorted(repos)) if repos else "-" + + +def render_report(spreads, promote, gaps, ledger): + """Join the live passes against the curated ledger into the checked-in burn-down markdown. + + A recorded disposition still matching a live divergence is a burn-down row. A live divergence (a + verbatim hand-modification, or a manifest gap) with no disposition reads UNTRIAGED. A disposition is + resolved only when every recorded repo now matches the canonical - a repo that went unavailable + (deleted, renamed, or too large to fetch inline) is unverified, not resolved, so it stays on the row. + Verbatim stale copies are the mechanical re-vendor list (the audit already flags them), kept separate. + """ + spread_by_path = {e["path"]: sp for e, sp in spreads if sp is not None} + + def buckets(path): + # (still divergent, confirmed match, unavailable) repo sets for a unit. Unavailable is held apart + # from match: an absent copy cannot confirm a divergence was fixed. + sp = spread_by_path.get(path) + if not sp: + return set(), set(), set() + return set(sp["differs"]) | set(sp["stale"]), set(sp["match"]), set(sp["unavailable"]) + + # Keep only well-formed entries so --report degrades cleanly on a hand-malformed ledger instead of + # raising KeyError/TypeError downstream. validate.py reports the malformation loudly in CI. + dispositions = [d for d in ledger.get("dispositions", []) + if isinstance(d, dict) and isinstance(d.get("path"), str) + and isinstance(d.get("repos"), list) and isinstance(d.get("disposition"), str) + and isinstance(d.get("reason"), str)] + gap_entries = [g for g in ledger.get("gaps", []) + if isinstance(g, dict) and isinstance(g.get("path"), str) + and isinstance(g.get("disposition"), str) and isinstance(g.get("reason"), str)] + gap_disp = {g["path"]: g for g in gap_entries} + covered = {} # path -> repos that carry a disposition (to find the untriaged remainder) + for d in dispositions: + covered.setdefault(d["path"], set()).update(d["repos"]) + + # Untriaged: verbatim hand-modifications with no disposition, and live gaps with no gap disposition. + # Verbatim-only by design: an intent unit's byte diff is expected (judged by meaning), so only a verbatim + # hand-modification with no recorded disposition is a genuine anomaly worth surfacing. + untriaged_files = [] + for e, sp in spreads: + if sp is None or e["fidelity"] != "verbatim": + continue + rest = sorted(set(sp["differs"]) - covered.get(e["path"], set())) + if rest: + untriaged_files.append((e["path"], rest)) + untriaged_gaps = [g for g in gaps if g not in gap_disp] + + order = ["re-vendor", "upstream-candidate", "investigate", "track", "accepted"] + by_disp = {k: [] for k in order} + resolved = [] + for d in dispositions: + dset = set(d["repos"]) + div, matched, unavail = buckets(d["path"]) + live = sorted(dset & div) + gone = sorted(dset & matched) + unk = sorted(dset & unavail) + if not live and not unk: # every recorded repo now matches the canonical + resolved.append(d) + continue + by_disp.setdefault(d["disposition"], []).append((d, live, gone, unk)) + for g in gap_entries: # a gap disposition stays live while the file is still an untracked gap + if g["path"] in gaps: + by_disp.setdefault(g["disposition"], []).append((g, None, None, None)) + + out = [] + w = out.append + w("# Fleet divergence report") + w("") + w("Generated by `python3 spec/fidelity_honesty.py --report` - do not hand-edit. Curate dispositions in [`spec/divergences.json`][ledger] and regenerate. Each row reflects a repo's ground-truth branch at generation time. Git dates this file.") + w("") + + w("## Burn-down") + w("") + if not any(by_disp[k] for k in by_disp): + w("_Nothing recorded and live._") + w("") + for k in order: + rows = by_disp.get(k, []) + if not rows: + continue + w(f"### {k}") + w("") + for entry, live, gone, unk in rows: + trk = f" (tracking: {entry['tracking']})" if entry.get("tracking") else "" + if live is None: # a manifest-gap disposition (not repo-scoped) + w(f"- **{entry['path']}** (manifest gap){trk} - {entry['reason']}") + else: + extra = f" _(recorded {_fmt(gone)} now resolved)_" if gone else "" + extra += f" _(unavailable, unverified: {_fmt(unk)})_" if unk else "" + w(f"- **{entry['path']}** - {_fmt(live) if live else '(none live)'}{trk}{extra} - {entry['reason']}") + w("") + + w("## Untriaged - add a disposition to `spec/divergences.json`") + w("") + if not untriaged_files and not untriaged_gaps: + w("_None - every live divergence has a recorded disposition._") + w("") + else: + for path, repos in untriaged_files: + w(f"- **{path}** - hand-modified in {_fmt(repos)} (verbatim canonical)") + for g in untriaged_gaps: + w(f"- **{g}** - carried by the reference adopter but not in the manifest") + w("") + + w("## Mechanical re-vendor (verbatim stale copies)") + w("") + w("A past hub revision, not the current canonical - the audit already flags these as DRIFT. Copy the current file down. No judgment needed.") + w("") + stale_rows = [(e["path"], sp["stale"]) for e, sp in spreads + if sp is not None and e["fidelity"] == "verbatim" and sp["stale"]] + if not stale_rows: + w("_None._") + for path, repos in stale_rows: + w(f"- **{path}** ({len(repos)}): {_fmt(repos)}") + w("") + + w("## Promote candidates (intent uniform -> verbatim)") + w("") + if not promote: + w("_None - no intent unit is currently fleet-uniform with the canonical._") + for e, sp in promote: + w(f"- **{e['path']}**: {len(sp['match'])} match, {len(sp['stale'])} stale, 0 hand-modified") + w("") + + if resolved: + w("## Resolved (recorded but no longer live - remove from the ledger)") + w("") + for d in resolved: + w(f"- **{d['path']}** - {_fmt(d['repos'])} ({d['disposition']})") + w("") + + # Reference-link definitions live at the bottom of the document. + w("[ledger]: ../spec/divergences.json") + return "\n".join(out).rstrip() + "\n" + + +def main(): + argv = sys.argv[1:] + report_mode = "--report" in argv + positional = [a for a in argv if not a.startswith("-")] + ref_repo = positional[0] if positional else REF_ADOPTER + spec = { + "registry": audit.load("registry/repos.json"), + "files": audit.load("spec/files.json"), + } + # Fail loud on an unresolvable reference adopter: its empty gap section would read as a false clean. + if report_mode and not any(isinstance(r, dict) and r.get("name") == ref_repo + for r in spec["registry"].get("repos", [])): + print(f"error: reference adopter '{ref_repo}' not found in the registry - " + f"cannot run the manifest-gap pass for the report", file=sys.stderr) + return 1 + spreads, promote, mislabel = fidelity_pass(spec) + slug, gaps = manifest_gap_pass(spec, ref_repo) + + if report_mode: + content = render_report(spreads, promote, gaps, load_ledger()) + # CRLF to match the fleet default (reports/*.md is CRLF). Write bytes so the local platform does not + # re-translate. Git dates the file, so no timestamp is embedded (it would churn every regeneration). + (audit.ROOT / REPORT_PATH).write_bytes(content.replace("\n", "\r\n").encode("utf-8")) + print(f"Wrote {REPORT_PATH} ({len(content.splitlines())} lines)") + return 0 + + print("== Per-unit fleet spread (ground-truth branch per repo) ==") + print(" fidelity path :: match / stale / differs / unavailable (absent or non-inline)") + for e, spread in spreads: + if spread is None: + print(f" {e.get('fidelity'):8} {e['path']} :: canonical unreadable at hub - skipped") + continue + print(f" {e['fidelity']:8} {e['path']} :: " + f"{len(spread['match'])} / {len(spread['stale'])} / {len(spread['differs'])} / {len(spread['unavailable'])}") + + print("\n== INTENT units with no divergent copy (verbatim-appropriate) -> candidates to promote to VERBATIM ==") + print(" (>=1 confirmed match, 0 hand-modified. Any stale/unavailable copy is shown per unit and would") + print(" re-vendor under verbatim - the drift intent cannot catch)") + if not promote: + print(" none") + for e, spread in promote: + print(f" {e['path']}: {len(spread['match'])} match, {len(spread['stale'])} stale, " + f"{len(spread['unavailable'])} unavailable, 0 differ") + + print("\n== VERBATIM units with NON-stale downstream divergence (mis-label or real drift) ==") + if not mislabel: + print(" none") + for e, spread in mislabel: + print(f" {e['path']}: differs in {', '.join(spread['differs'])}") + + print(f"\n== Manifest gap: files carried by {ref_repo} + present at the hub but NOT in the manifest ==") + if slug is None: + print(f" {ref_repo} not found in the registry") + elif not gaps: + print(" none - the manifest covers every hub file the reference adopter also carries") + else: + for g in gaps: + print(f" UNTRACKED {g}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/spec/files.json b/spec/files.json index 4d0ca5fa..1833489c 100644 --- a/spec/files.json +++ b/spec/files.json @@ -9,14 +9,15 @@ { "path": "HISTORY.md", "appliesTo": "*" }, { "path": ".github/copilot-instructions.md", "fidelity": "intent", "whole": true, "placeholders": ["", "", ""], "appliesTo": "*" }, { "path": ".editorconfig", "fidelity": "intent", "whole": true, "intentRef": "AGENTS.md#line-endings", "appliesTo": "*" }, + { "path": ".editorconfig-checker.json", "fidelity": "intent", "whole": true, "intentRef": "AGENTS.md#line-endings", "appliesTo": "*" }, { "path": ".gitattributes", "fidelity": "intent", "whole": true, "intentRef": "AGENTS.md#line-endings", "appliesTo": "*" }, { "path": ".markdownlint-cli2.jsonc", "fidelity": "verbatim", "whole": true, "appliesTo": "*" }, { "path": "cspell.json", "fidelity": "intent", "whole": true, "appliesTo": "*" }, { "path": ".gitignore", "appliesTo": "*" }, { "path": "version.json", "fidelity": "intent", "intentRef": "WORKFLOW.md#d3---versioning-and-classification", "appliesTo": "*" }, - { "path": "repo-config/develop.json", "fidelity": "intent", "intentRef": "repo-config/README.md", "appliesTo": ["release"] }, - { "path": "repo-config/operational/develop.json", "fidelity": "intent", "intentRef": "repo-config/README.md", "appliesTo": ["operational"] }, - { "path": "repo-config/main.json", "fidelity": "intent", "intentRef": "repo-config/README.md", "appliesTo": "*" }, + { "path": "repo-config/develop.json", "fidelity": "verbatim", "whole": true, "appliesTo": ["release"] }, + { "path": "repo-config/operational/develop.json", "fidelity": "verbatim", "whole": true, "appliesTo": ["operational"] }, + { "path": "repo-config/main.json", "fidelity": "verbatim", "whole": true, "appliesTo": "*" }, { "path": "repo-config/README.md", "fidelity": "intent", "whole": true, "intentRef": "repo-config/README.md", "appliesTo": "*" }, { "path": "repo-config/configure.sh", "fidelity": "verbatim", "whole": true, "appliesTo": "*" }, { "path": "repo-config/settings.json", "fidelity": "intent", "whole": true, "intentRef": "repo-config/README.md", "appliesTo": "*" }, diff --git a/spec/project-types.json b/spec/project-types.json index 9f51747b..6df7f80d 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -40,7 +40,7 @@ { "id": "python.mypy.allowed", "verdict": "intent", "assert": "mypy is permitted as an additional type checker (not banned); required for a Home Assistant integration (platinum strict-typing) and is the SCRIPTS profile's type checker. When used it runs in CI and the editor.", "intentRef": "CODESTYLE.md" }, { "id": "python.coverage.codecov", "verdict": "letter", "assert": "The test job collects coverage (pytest --cov-report=xml) and uploads it to Codecov via codecov/codecov-action, best-effort (continue-on-error and fail_ci_if_error: false); CODECOV_TOKEN is stored in the repo actions secrets. Required for every Python repo with tests. N/A for the SCRIPTS profile (lint/type-checked only, no pytest); in a mixed repo the codecov.yml file-presence is still required by any co-present type that has tests, e.g. csharp.", "intentRef": "WORKFLOW.md" }, { "id": "python.uvlock.pinned", "verdict": "letter", "assert": "PROJECT profile: the committed uv.lock is pinned to LF in both .editorconfig ([uv.lock]) and .gitattributes (uv.lock text eol=lf); uv regenerates it LF on every platform, so a CRLF-default repo otherwise reds editorconfig-checker on every uv lock/sync. N/A for a non-uv Python repo (a Home Assistant integration on pip/requirements) and for the SCRIPTS profile (no uv.lock by definition).", "intentRef": "AGENTS.md#line-endings" }, - { "id": "python.scripts.uvx", "verdict": "letter", "assert": "SCRIPTS profile only: the tools run via uvx (no project install, no lockfile). CI pins exact tool versions in the uvx command (e.g. uvx ruff@X, uvx mypy@Y), bumpable there; the VS Code tasks and README run the unpinned latest, a deliberate CI-vs-local gap so local never silently falls behind CI. N/A for the PROJECT profile (which pins tool versions via uv.lock + uv sync --frozen instead).", "intentRef": "CODESTYLE.md" } + { "id": "python.scripts.uvx", "verdict": "letter", "assert": "SCRIPTS profile only: the tools run via uvx (no project install, no lockfile). A uvx @ pin in a run: step is not Dependabot-trackable, so CI runs uvx ruff@latest / uvx mypy@latest - the fleet rule pins only what Dependabot auto-updates and otherwise runs latest, never a manual pin that goes stale. VS Code tasks, README, and CI all run the unpinned latest. N/A for the PROJECT profile (which pins tool versions via uv.lock + uv sync --frozen instead).", "intentRef": "CODESTYLE.md" } ] }, "console": { diff --git a/spec/validate.py b/spec/validate.py index 668d6b2c..ed0032e7 100644 --- a/spec/validate.py +++ b/spec/validate.py @@ -289,6 +289,64 @@ def check_selector(where, applies_to): elif not isinstance(elt, str): errors.append(f"files.json: {path} section entry {elt!r} must be a string or object") + # Validate the divergence ledger (spec/divergences.json) when present, so a mistyped repo name or + # disposition fails CI instead of silently dropping a burn-down row. + dispositions = ("re-vendor", "track", "accepted", "upstream-candidate", "investigate") + if (ROOT / "spec/divergences.json").exists(): + div = load("spec/divergences.json") + repo_names = {r.get("name") for r in repos["repos"] if isinstance(r, dict)} + manifest_paths = {i.get("path") for i in baseline if isinstance(i, dict)} + # Guard the root type: a non-object root (a list from a bad edit) would crash the .get() calls below. + if not isinstance(div, dict): + errors.append("divergences.json: root must be an object") + div = {} + div_dispositions = div.get("dispositions", []) + if not isinstance(div_dispositions, list): + errors.append("divergences.json: 'dispositions' must be an array") + div_dispositions = [] + div_gaps = div.get("gaps", []) + if not isinstance(div_gaps, list): + errors.append("divergences.json: 'gaps' must be an array") + div_gaps = [] + for d in div_dispositions: + if not isinstance(d, dict): + errors.append(f"divergences.json: disposition {d!r} is not an object") + continue + p = d.get("path") + # isinstance guard first: a non-string path is unhashable and would crash the membership test. + if not isinstance(p, str): + errors.append(f"divergences.json: disposition path {p!r} must be a string") + elif p not in manifest_paths: + errors.append(f"divergences.json: disposition path '{p}' is not a manifest unit") + if d.get("disposition") not in dispositions: + errors.append(f"divergences.json: '{p}' disposition '{d.get('disposition')}' invalid (expected one of {', '.join(dispositions)})") + if not is_str_list(d.get("repos")) or not d.get("repos"): + errors.append(f"divergences.json: '{p}' repos must be a non-empty array of strings") + else: + for rn in d["repos"]: + if rn not in repo_names: + errors.append(f"divergences.json: '{p}' repo '{rn}' not in the registry") + if not isinstance(d.get("reason"), str) or not d.get("reason"): + errors.append(f"divergences.json: '{p}' reason must be a non-empty string") + if not (d.get("tracking") is None or isinstance(d.get("tracking"), str)): + errors.append(f"divergences.json: '{p}' tracking must be a string or null") + for g in div_gaps: + if not isinstance(g, dict): + errors.append(f"divergences.json: gap {g!r} is not an object") + continue + gp = g.get("path") + # isinstance guard first: a non-string path is unhashable and would crash the membership test. + if not isinstance(gp, str): + errors.append(f"divergences.json: gap path {gp!r} must be a string") + elif gp in manifest_paths: + errors.append(f"divergences.json: gap '{gp}' is already a manifest unit (not a gap)") + if g.get("disposition") not in dispositions: + errors.append(f"divergences.json: gap '{gp}' disposition '{g.get('disposition')}' invalid (expected one of {', '.join(dispositions)})") + if not isinstance(g.get("reason"), str) or not g.get("reason"): + errors.append(f"divergences.json: gap '{gp}' reason must be a non-empty string") + if not (g.get("tracking") is None or isinstance(g.get("tracking"), str)): + errors.append(f"divergences.json: gap '{gp}' tracking must be a string or null") + if errors: print("Spec validation FAILED:") for e in errors: