From ddec2d641a580fa0b47e0ed6db60ef43bdd3f210 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 3 Sep 2026 11:21:55 -0700 Subject: [PATCH 1/4] Settle the Coverage-Applicability Boundary and Scope D7.3 to Event Inputs Tests are the trigger for D1.6, not the dependency mechanism. The hub validator gated its Python leg on uv.lock, so a pip/requirements repo with tests was skipped though the contract owed it coverage, while spec/audit.py claimed the Codecov requirement from the declared profile alone, so a package-only build repo was told to store a token for a report its pipeline never produces. Both sides of that boundary move together. The four Python unit-test steps now key on pyproject.toml, tests/, and a dependency manifest the leg can install from, a committed uv.lock or a root requirements*.txt, and the sync and pytest steps branch on which one is present. A repo carrying neither still skips, so no repo newly fails. coverage_claiming_types() reads the repo tree for tests using the validator's own detectors, and an unreadable tree keeps the claim. D7.3 said, unconditionally, that a boolean input is declared in both trigger blocks and compared against true and 'true'. Followed literally for smoke it inverted D1.3. It is now scoped to github.event.inputs, and the comparison is against 'true' alone: measured against GitHub's own expression evaluator, a non-numeric string casts to NaN and the boolean to 1, so the == true half is false even on the run where the input arrived as true. Fixes #1221 Fixes #1224 Fixes #1218 Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/validate-task.yml | 44 +++++++++--- WORKFLOW.md | 10 +-- docs/reusable-workflows.md | 2 +- reports/canonical-review.json | 16 ++--- scripts/tests/test_release_guards.py | 95 +++++++++++++++++++++++++ spec/audit.py | 100 +++++++++++++++++++++++---- spec/project-types.json | 2 +- spec/type-model.md | 4 +- 8 files changed, 235 insertions(+), 38 deletions(-) diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index b2aae10e..8b61da20 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -1,7 +1,7 @@ name: Validate task # The fleet validation gate, hosted here once and reached by every repo's test-pull-request stub and its own publish-release stub. -# Three jobs: lint (the fleet doc-lint block plus language lint by tree detection, the prose gate, and the repo gate), unit-test (a generic dotnet test or uv run pytest, skipped where the caller has no test project), and validate (the validate hook, a repo's own domain checks such as an ESPHome compile, a Hugo build, a KiCad ERC, a codegen-drift check, or PowerShell tests). +# Three jobs: lint (the fleet doc-lint block plus language lint by tree detection, the prose gate, and the repo gate), unit-test (a generic dotnet test or pytest, skipped where the caller has no test project), and validate (the validate hook, a repo's own domain checks such as an ESPHome compile, a Hugo build, a KiCad ERC, a codegen-drift check, or PowerShell tests). # No permissions beyond contents: read where a job needs one, since every job here only checks out and reads. # No required inputs, markdown-exclude-globs and repo-gate-exclude-globs are the two optional inputs, and CODECOV_TOKEN is the one optional secret, since coverage upload is best-effort. # Hub-owned gates and default hooks resolve through $/ at the reusable workflow's commit, so each implementation is reproducible against the caller's released pin without a second checkout. @@ -257,8 +257,9 @@ jobs: # No job-level if: here, since GitHub Actions does not evaluate hashFiles in a job condition, only a step one. # Every step below carries its own tree-detection guard instead. - # A caller with neither a *Tests*.csproj nor a uv.lock-backed tests/ directory beside a pyproject.toml runs every step's guard false, and the job reports success having done nothing, which is the clean skip this job promises. - # The uv.lock guard excludes the lint-only Python profile (spec/project-types.json python profileNote), which is stdlib-only, uvx-run, and carries no lockfile to sync from. + # A caller with neither a *Tests*.csproj nor a tests/ directory beside a pyproject.toml and a dependency manifest runs every step's guard false, and the job reports success having done nothing, which is the clean skip this job promises. + # That manifest is a committed uv.lock or a root requirements*.txt, the two dependency mechanisms spec/project-types.json names, so a pip-based Python repo with tests is served here rather than skipped, per D1.6. + # The lint-only Python profile (spec/project-types.json python profileNote) carries neither, being stdlib-only and uvx-run, so it stays excluded. unit-test: name: Unit test job runs-on: ubuntu-latest @@ -299,24 +300,47 @@ jobs: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} - name: Setup uv step - if: hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' && hashFiles('uv.lock') != '' + if: >- + hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' + && (hashFiles('uv.lock') != '' || hashFiles('requirements*.txt') != '') uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 with: python-version: "3.13" + # The dependency mechanism is read from the tree rather than assumed: a committed uv.lock is the uv project shape and syncs frozen, and a root requirements*.txt is the pip shape a non-uv Python repo carries instead. + # The pip branch installs into the .venv that uv pip install resolves by default, so the pytest step below can name that interpreter directly and needs no activation carried between steps. + # Every requirements*.txt is installed rather than one canonical name, since the test dependencies live in a second file whose spelling differs by repo. - name: Sync dependencies step - if: hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' && hashFiles('uv.lock') != '' - run: uv sync --all-groups --frozen + if: >- + hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' + && (hashFiles('uv.lock') != '' || hashFiles('requirements*.txt') != '') + run: | + set -Eeuo pipefail + if [ -f uv.lock ]; then + uv sync --all-groups --frozen + else + uv venv + for file in requirements*.txt; do + [ -e "$file" ] || continue + uv pip install -r "$file" + done + fi # --cov-report=xml names the report format and selects nothing to measure, so the repository's own pyproject.toml supplies the --cov selector, per D1.6. # The report is checked rather than assumed, because the best-effort upload below reads a missing file exactly as it reads a healthy run. # The pre-run delete makes that a check on what this run wrote, since a committed coverage.xml would otherwise satisfy it without any measurement. - name: Run pytest step - if: hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' && hashFiles('uv.lock') != '' + if: >- + hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' + && (hashFiles('uv.lock') != '' || hashFiles('requirements*.txt') != '') run: | set -Eeuo pipefail rm -f coverage.xml - uv run pytest --cov-report=xml + if [ -f uv.lock ]; then + uv run pytest --cov-report=xml + else + .venv/bin/python -m pytest --cov-report=xml + fi if [[ ! -s coverage.xml ]]; then echo "::error::This run wrote no coverage.xml at the repository root. Select a coverage source in this repository's pyproject.toml, an addopts entry of --cov= in practice, since --cov-report=xml alone measures nothing, and leave the report at the root path the upload step below reads." exit 1 @@ -324,7 +348,9 @@ jobs: # Best-effort: continue-on-error plus fail_ci_if_error false, so a missing token never reds the gate. - name: Upload coverage to Codecov step (Python) - if: hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' && hashFiles('uv.lock') != '' + if: >- + hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' + && (hashFiles('uv.lock') != '' || hashFiles('requirements*.txt') != '') continue-on-error: true uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: diff --git a/WORKFLOW.md b/WORKFLOW.md index 371d44c7..60b81313 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -150,11 +150,11 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as the o ### D1 - PR Fast-Feedback (Smoke) - **D1.1 Only changed targets build.** Input: a PR touching some targets. Output: the paths-filter marks exactly those targets and only their smoke builds run. Unchanged targets skip. A repo's own targets MUST each have a filter entry (so a touched target is never silently skipped), and that entry lists paths rather than negating them, so a change matching no entry marks nothing and every smoke build skips. A filter written the other way round, as a negation of the paths that must not build, marks a docs-only change as a target change: it satisfies D1.4 and violates this item. *Prevents: rebuilding everything, and a changed target slipping through unbuilt.* -- **D1.2 A validation job always runs.** Input: any PR. Output: a validation job runs unconditionally and the aggregator `needs:` it. That job is the caller's own job reaching the reusable validator, named `validate` in every shipped stub, and that name is what the aggregator's `needs:` carries. The validator's internal jobs (`lint`, `unit-test` and `validate` in the hub's `validate-task.yml`) are not addressable from a caller, so a `validate` in a caller's `needs:` list always names the caller's own job rather than the validator's internal one of the same name. The validator detects the tree rather than the repo's language, running the doc and repo gates everywhere and the `dotnet test` or `uv run pytest` path only where that tree is present, so a non-.NET repo calls the same one rather than replacing it. A repo whose validation it cannot express **replaces** the call (not deletes it) with its own validator and re-points the aggregator's `needs:` to the replacement. `smoke-build` `needs:` the `changes` job rather than the validation job, so no second `needs:` moves with it. *Prevents: a PR merging with no validation, or a dangling `needs:` that stops the whole workflow from loading.* +- **D1.2 A validation job always runs.** Input: any PR. Output: a validation job runs unconditionally and the aggregator `needs:` it. That job is the caller's own job reaching the reusable validator, named `validate` in every shipped stub, and that name is what the aggregator's `needs:` carries. The validator's internal jobs (`lint`, `unit-test` and `validate` in the hub's `validate-task.yml`) are not addressable from a caller, so a `validate` in a caller's `needs:` list always names the caller's own job rather than the validator's internal one of the same name. The validator detects the tree rather than the repo's language, running the doc and repo gates everywhere and the `dotnet test` or `pytest` path only where that tree is present, so a non-.NET repo calls the same one rather than replacing it. A repo whose validation it cannot express **replaces** the call (not deletes it) with its own validator and re-points the aggregator's `needs:` to the replacement. `smoke-build` `needs:` the `changes` job rather than the validation job, so no second `needs:` moves with it. *Prevents: a PR merging with no validation, or a dangling `needs:` that stops the whole workflow from loading.* - **D1.3 Smoke never publishes and never uploads.** Input: `smoke: true`. Output: full compile/lint/test, but no registry/image push, no release, and **no** artifact uploads (every `upload-artifact`, including any aggregation job, is gated on smoke being false, written `!inputs.smoke` at the workflow layer and `inputs.smoke != 'true'` in a composite action, whose inputs are strings). *Prevents: a PR publishing, and orphaned artifacts churning the storage quota.* - **D1.4 Workflow-file changes are not smoke-built.** Input: a PR changing only `.github/workflows/**`. Output: the paths-filter marks no target, so smoke-build skips. An inclusion list satisfying D1.1 reaches this by leaving workflow paths out of every target's entry. *Implication: a workflow-only change is not smoke-built, but actionlint still validates it in CI.* - **D1.5 One required aggregator gates merge.** Input: any PR. Output: a single aggregator job must **succeed**, run under `if: always()` so a failed or skipped dependency cannot skip the gate itself, `needs:` the validation job, and the `changes` and `smoke-build` jobs too wherever the repo has a smoke build, treat a **skipped** smoke build as pass, and **block** on `failure`/`cancelled`. Its name is ruleset-bound: the job `name:` and the ruleset `context:` are the same string and MUST be renamed together, never independently. *Prevents: a paths-filter error letting a target-changing PR merge unbuilt.* -- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --coverage --coverage-output-format cobertura --results-directory ./coverage` or `pytest --cov-report=xml` over a repo whose own pytest configuration selects what to measure) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** and **dependabot** secret stores, the second because a run triggered by a Dependabot PR reads the Dependabot store and the upload would otherwise skip silently on every bot PR, and a caller passing it names it (`secrets:` with `CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}`), the way every hub task's declared secrets are passed, since `secrets: inherit` is documented for a caller in the same organization or enterprise and this fleet is a personal account, so a cross-repository call names each secret it passes. A call by local path stays inside one repository and may inherit instead. The publisher stub's validation job names the secret and every pull request stub's validation job passes no `secrets:` key at all, so a repo whose coverage must reach Codecov from its pull requests adds the mapping there itself. Required for **every** C# and Python repo that has tests. That C# invocation runs under **Microsoft.Testing.Platform**, which an MTP-based test project on the .NET 10 SDK and later requires, since running one through the VSTest target fails outright. A repo whose test project is MTP-based, in practice any repo on xunit.v3 4.0.0 or later, therefore also ships a root **`global.json`** declaring `{"test": {"runner": "Microsoft.Testing.Platform"}}`, references **`Microsoft.Testing.Extensions.CodeCoverage`** at **18.9.0 or later** in place of `coverlet.collector`, whose VSTest data collector MTP ignores without failing, and drops `xunit.runner.visualstudio`, the VSTest adapter MTP replaces. A repo whose test project is not yet MTP-based keeps the VSTest collector and its existing pin on the reusable validator, and that lagging state is a migration still owed rather than drift, until its own bump makes the project MTP-based and forces the move. The version floor is load-bearing rather than cautionary. Below 18.1.0 the extension is built against Microsoft.Testing.Platform 1.x, and an 18.0.x resolution, which is what a `>= 18.0.0` range picks, throws a `TypeLoadException` against the 2.x platform xunit.v3 4.0.0 carries, runs zero tests, and still writes a well-formed Cobertura file reporting full coverage, so only the non-zero exit says the run reported nothing. 18.9.0 is the first release on Microsoft.Testing.Platform 2.3.x, where every test project writes into the one shared `--results-directory` the invocation names rather than resolving that relative path per project. Two details of the invocation are equally load-bearing, and neither failure reds the job on its own. `--coverage-output` stays unset, because pinning one filename gives every test project in the solution the same path and a repo with more than one then keeps only whichever ran last. Leaving it unset produces the default name `.cobertura.xml`, which `codecov-cli`'s own file finder does not match. The validator therefore prefixes each report to `coverage-.cobertura.xml` before the upload step reads the directory. The Python invocation carries a load-bearing detail of its own, an omission rather than a collision: it names the report format and selects nothing to measure. `pytest-cov` reports on what `--cov` selects, so `--cov-report=xml` on its own measures nothing, writes no file, and exits zero, which the best-effort upload then reads exactly as it reads a healthy run. A Python repo with tests therefore references **`pytest-cov`** in its dev dependency group, selects the coverage source in its own `pyproject.toml`, an `addopts` entry of `--cov=` in practice, and leaves the report at the repo root as `coverage.xml`, already the one path the upload step names. The validator **fails the test step when that file was not written**, since nothing downstream of it can tell an absent report from an uploaded one, so a repo that redirects the report through `[tool.coverage.xml]` reds the gate rather than uploading nothing from a green run. That step sits in the hub validator's Python leg, which runs where the repository root carries `pyproject.toml`, `tests/`, and `uv.lock`. Where this guarantee does not apply (a `lint-only` profile for that type, per the hub's `registry/repos.json`), the hub's `spec/secrets.json` `typeMechanisms` mapping is not claimed for that repo, and the absence is not drift. The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR (a distinct knob from `fail_ci_if_error`, which only guards the upload step), and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact, so `.gitignore` excludes it (e.g. `coverage/`, `coverage.xml`, and `*.cobertura.xml`, with `.gitignore` the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported, a test project stranded on a runner the current SDK refuses, a stale and unused token, a coverage regression blocking an unrelated PR, and a coverage artifact committed by a blanket add.* +- **D1.6 Coverage is reported to Codecov (C# and Python).** Input: a C# or Python repo's validation/test job. Output: tests run with coverage collection (`dotnet test --coverage --coverage-output-format cobertura --results-directory ./coverage` or `pytest --cov-report=xml` over a repo whose own pytest configuration selects what to measure) and a `codecov/codecov-action` step uploads it, **best-effort** (`continue-on-error` and/or `fail_ci_if_error: false`, so a Codecov outage or an absent token never reds the gate). `CODECOV_TOKEN` lives in the repo's **actions** and **dependabot** secret stores, the second because a run triggered by a Dependabot PR reads the Dependabot store and the upload would otherwise skip silently on every bot PR, and a caller passing it names it (`secrets:` with `CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}`), the way every hub task's declared secrets are passed, since `secrets: inherit` is documented for a caller in the same organization or enterprise and this fleet is a personal account, so a cross-repository call names each secret it passes. A call by local path stays inside one repository and may inherit instead. The publisher stub's validation job names the secret and every pull request stub's validation job passes no `secrets:` key at all, so a repo whose coverage must reach Codecov from its pull requests adds the mapping there itself. Required for **every** C# and Python repo that has tests. That C# invocation runs under **Microsoft.Testing.Platform**, which an MTP-based test project on the .NET 10 SDK and later requires, since running one through the VSTest target fails outright. A repo whose test project is MTP-based, in practice any repo on xunit.v3 4.0.0 or later, therefore also ships a root **`global.json`** declaring `{"test": {"runner": "Microsoft.Testing.Platform"}}`, references **`Microsoft.Testing.Extensions.CodeCoverage`** at **18.9.0 or later** in place of `coverlet.collector`, whose VSTest data collector MTP ignores without failing, and drops `xunit.runner.visualstudio`, the VSTest adapter MTP replaces. A repo whose test project is not yet MTP-based keeps the VSTest collector and its existing pin on the reusable validator, and that lagging state is a migration still owed rather than drift, until its own bump makes the project MTP-based and forces the move. The version floor is load-bearing rather than cautionary. Below 18.1.0 the extension is built against Microsoft.Testing.Platform 1.x, and an 18.0.x resolution, which is what a `>= 18.0.0` range picks, throws a `TypeLoadException` against the 2.x platform xunit.v3 4.0.0 carries, runs zero tests, and still writes a well-formed Cobertura file reporting full coverage, so only the non-zero exit says the run reported nothing. 18.9.0 is the first release on Microsoft.Testing.Platform 2.3.x, where every test project writes into the one shared `--results-directory` the invocation names rather than resolving that relative path per project. Two details of the invocation are equally load-bearing, and neither failure reds the job on its own. `--coverage-output` stays unset, because pinning one filename gives every test project in the solution the same path and a repo with more than one then keeps only whichever ran last. Leaving it unset produces the default name `.cobertura.xml`, which `codecov-cli`'s own file finder does not match. The validator therefore prefixes each report to `coverage-.cobertura.xml` before the upload step reads the directory. The Python invocation carries a load-bearing detail of its own, an omission rather than a collision: it names the report format and selects nothing to measure. `pytest-cov` reports on what `--cov` selects, so `--cov-report=xml` on its own measures nothing, writes no file, and exits zero, which the best-effort upload then reads exactly as it reads a healthy run. A Python repo with tests therefore declares **`pytest-cov`** among its test dependencies, a dev dependency group where the repo is a uv project and a `requirements*.txt` entry where it is on pip, selects the coverage source in its own `pyproject.toml`, an `addopts` entry of `--cov=` in practice, and leaves the report at the repo root as `coverage.xml`, already the one path the upload step names. The validator **fails the test step when that file was not written**, since nothing downstream of it can tell an absent report from an uploaded one, so a repo that redirects the report through `[tool.coverage.xml]` reds the gate rather than uploading nothing from a green run. That step sits in the hub validator's Python leg, which runs where the repository root carries `pyproject.toml`, `tests/`, and a dependency manifest the leg can install from, being a committed `uv.lock` or a root `requirements*.txt`. Those are the two dependency mechanisms the hub's `spec/project-types.json` names, and the leg reads the tree for either rather than for the lockfile alone, so a pip-based Python repo with tests is served here rather than skipped. Tests are what this guarantee turns on: a repo carrying no tests for that type owes no coverage whatever its dependency mechanism, since a token and a `codecov.yml` would then gate on a report nothing produces. A `lint-only` profile for that type (per the hub's `registry/repos.json`) owes none either, whatever tests it carries, since nothing there is built or packaged to report on. Where the guarantee does not apply, the hub's `spec/secrets.json` `typeMechanisms` mapping is not claimed for that repo, and the absence is not drift. The repo also ships a **`codecov.yml`** that sets the project and patch statuses to **`informational: true`** so a coverage delta never gates a PR (a distinct knob from `fail_ci_if_error`, which only guards the upload step), and excludes intentionally-untested, non-shipped code (an example/demo or benchmark project) from the coverage denominator via `ignore`. A repo may override this to enforce a coverage threshold where its quality bar requires it. Coverage output is a build artifact, so `.gitignore` excludes it (e.g. `coverage/`, `coverage.xml`, and `*.cobertura.xml`, with `.gitignore` the full source of truth) so a blanket `git add -A` won't stage the untracked output. *Prevents: coverage silently going unreported, a test project stranded on a runner the current SDK refuses, a stale and unused token, a coverage regression blocking an unrelated PR, and a coverage artifact committed by a blanket add.* ### D2 - Input/State Validation at Entry @@ -200,7 +200,7 @@ The required behaviors, organized by domain. Each is a **MUST**, stated as the o - **D7.1 Publisher serializes.** Output: the publisher uses a **global, ref-independent** concurrency group with `cancel-in-progress: false`. *Prevents: a schedule and a dispatch double-pushing, or a cancelled publish leaving a partial release.* - **D7.2 A called job's permissions block is validated before its `if:`.** Output: a reusable job declares `permissions:` only where **every** caller grants that scope at startup, and otherwise declares none and runs under whatever the calling job granted. A callee's extra scope (e.g. `actions: write` for cleanup, or `id-token: write` for OIDC) is granted by the caller and appears at exactly the one entry point that needs it. *Prevents: a `startup_failure` on every caller that does not grant a scope only one target needs, including a smoke build under a read-only pull request token.* -- **D7.3 Boolean inputs both forms.** Output: declared in both trigger blocks, compared against `true` and `'true'`. +- **D7.3 A `github.event.inputs` boolean is compared as a string.** Output: a boolean read through `github.event.inputs.` is compared against `'true'`, since that context delivers every input as a string whatever the input's declared type. Comparing it against the boolean `true` as well is dead rather than defensive: an operand-type mismatch casts each side to a number, a non-numeric string casts to `NaN`, and `NaN` compares equal to nothing, so `github.event.inputs. == true` is false even on the run where the input arrived as `true`. The `inputs` context preserves the declared boolean on the `workflow_call` and `workflow_dispatch` paths alike, so an `inputs.` read is used directly, and a both-forms comparison there is redundant rather than wrong, which is why the hub's Docker build task comparing its `build-base` input in both forms is not a finding. A workflow carrying both trigger blocks declares each boolean input in both, since one declaration does not propagate to the other, while a boolean that only ever arrives by `workflow_call` is declared in that block alone. `smoke` is such a boolean, every hub task declaring it being `workflow_call`-only, which is why D1.3 writes the workflow-layer gate `!inputs.smoke` against the real boolean and the composite-action gate `inputs.smoke != 'true'` against a string, a composite action's inputs being strings whatever their caller passed. A job or step **output** is a string for the same reason and takes the same `== 'true'` rather than a bare truthiness test, since the string `'false'` is truthy. *Prevents: a dispatch-path string read as truthy, and a comparison against the boolean `true`, which can never fire, standing in for the one that can.* - **D7.4 Optional-dependency chaining.** Output: cross-job conditions allowlist `success`/`skipped` explicitly, paired with a status-check function such as `always()` or `!failure() && !cancelled()`. Without one the implicit `success()` applies and is false the moment any `needs:` job skipped, which is the case the allowlist exists to admit. *Prevents: a condition that reads as tolerant of a skipped dependency and is dead in exactly that case.* ### D8 - Bots / Automation @@ -229,13 +229,13 @@ Read the workflow files, `version.json`, and whatever else a check below names a **Core (every repo):** - **D1:** a `changes` paths-filter job exists wherever the repo has a smoke build, with one entry per target naming the paths that target is built from, so a change touching no target marks nothing (a filter written as a negation instead marks a docs-only change and fails D1.1); the PR entry workflow's smoke call sets every publish flag its release task declares to false (`github`/`dockerhub`, and a package-push flag there is itself a finding, per section 1); a pushing leaf receives `smoke: true` and a derived `push` (false on smoke), and a build-only leaf receives `smoke: true` with no `push` to derive; every `upload-artifact` the smoke call reaches, in a build task and in any job collecting other jobs' artifacts alike, is gated on smoke being false, written `!inputs.smoke` at the workflow layer and `inputs.smoke != 'true'` in a composite action, whose inputs are strings; the aggregator runs under `if: always()`, `needs:` the validation job, and the `changes` and `smoke-build` jobs too wherever the repo has a smoke build, blocks on `failure`/`cancelled`, and passes on a **skipped smoke build**, so a no-build repo's aggregator, having only the validation job to read, requires that job to have succeeded; the aggregator's own job `name:` is the string the branch ruleset's required-check `context:` carries; a validation job runs unconditionally. -- **D1.6:** the validator the repo's validation job reaches collects coverage and uploads it, in every C# and Python repo that has tests. Its C# leg runs `dotnet test --coverage --coverage-output-format cobertura --results-directory ./coverage`, with `--coverage-output` unset, wherever the repo's test project is MTP-based. An MTP-based repo also ships a root `global.json` declaring the Microsoft.Testing.Platform runner, references `Microsoft.Testing.Extensions.CodeCoverage` at 18.9.0 or later in place of `coverlet.collector`, and carries no `xunit.runner.visualstudio`, while a repo still on the VSTest collector keeps its existing validator pin, a migration owed rather than drift. Its Python leg runs `pytest --cov-report=xml`, and the repo references `pytest-cov` in its dev dependency group, selects the coverage source in its own `pyproject.toml` rather than leaving `--cov` unset, and leaves the report at the repo root as `coverage.xml`. Either way the report reaches a `codecov/codecov-action` step made best-effort by `continue-on-error` and/or `fail_ci_if_error: false`, and `CODECOV_TOKEN` is present in **both** the repo's Actions and its Dependabot secret names, the second because a Dependabot-triggered run reads that store and the upload otherwise skips silently on every bot pull request. `codecov.yml` sets the project and patch statuses `informational: true`, or names the threshold the repo enforces instead, and lists any intentionally-untested, non-shipped project under `ignore`. `.gitignore` excludes the coverage output. Record the whole item N/A for a repo with no tests, and for a type the audited repo carries at the `lint-only` profile. +- **D1.6:** the validator the repo's validation job reaches collects coverage and uploads it, in every C# and Python repo that has tests. Its C# leg runs `dotnet test --coverage --coverage-output-format cobertura --results-directory ./coverage`, with `--coverage-output` unset, wherever the repo's test project is MTP-based. An MTP-based repo also ships a root `global.json` declaring the Microsoft.Testing.Platform runner, references `Microsoft.Testing.Extensions.CodeCoverage` at 18.9.0 or later in place of `coverlet.collector`, and carries no `xunit.runner.visualstudio`, while a repo still on the VSTest collector keeps its existing validator pin, a migration owed rather than drift. Its Python leg runs `pytest --cov-report=xml`, and the repo declares `pytest-cov` among its test dependencies, a dev dependency group in a uv project and a `requirements*.txt` entry on pip, selects the coverage source in its own `pyproject.toml` rather than leaving `--cov` unset, and leaves the report at the repo root as `coverage.xml`. Either way the report reaches a `codecov/codecov-action` step made best-effort by `continue-on-error` and/or `fail_ci_if_error: false`, and `CODECOV_TOKEN` is present in **both** the repo's Actions and its Dependabot secret names, the second because a Dependabot-triggered run reads that store and the upload otherwise skips silently on every bot pull request. `codecov.yml` sets the project and patch statuses `informational: true`, or names the threshold the repo enforces instead, and lists any intentionally-untested, non-shipped project under `ignore`. `.gitignore` excludes the coverage output. Record the whole item N/A for a repo with no tests, and for a type the audited repo carries at the `lint-only` profile. - **D2:** an entry validation job/step exists per complex-input workflow; the release gate checks both directions, strips `+buildmetadata`, and skips on smoke; the publisher rejects a dispatch from a ref other than `main` or `develop`. - **D3:** each run builds one branch, so NBGV classifies `github.ref` directly (no `IGNORE_GITHUB_REF`), and the default-branch literal in the gate (`== 'main'`), the `prerelease` expression (`!= 'main'`), and `version.json`'s `publicReleaseRefSpec` all name the repo's actual default branch. `version.json` sets the major.minor floor, and NBGV and `version.json` are both retained even by a repo with no compiler, since they own the tag (D3.3). NuGet.org derives the prerelease flag from the SemVer2 `-g` suffix rather than the workflow setting one, and the PyPI version is built from `AssemblyFileVersion` with `.dev0` appended on `develop` only (D3.4). - **D4:** `target_commitish` is the NBGV commit id; `prerelease` equals `branch != default`; the release-create step is gated `exists == 'false' || github.event_name == 'workflow_dispatch'` (the step output is the string `'false'`, not a boolean); the asset-delete step carries that same condition, narrowed by `inputs.expect_release_assets`. Where `workflow_dispatch` is the publisher's only trigger (`releaseTrigger: dispatch-only`) every run is a dispatch, so the exists-check's skip leg can never fire: record that leg N/A rather than failed, and expect the release-create step to still carry the `exists == 'false' || github.event_name == 'workflow_dispatch'` condition, since D6.4 keeps the `github-release` job body verbatim. A caller with no file target passes `expect_release_assets: false`, which covers a Docker-only, a PyPI-only, and a source-only repo alike, while a NuGet-only caller keeps the default `true` because its leaf uploads a `release-asset-*` carrying the package, and a source-only caller also sets every `enable_*` input false. `github-release` and the terminal registry pusher (Docker) each carry `!failure() && !cancelled()` rather than the implicit `success()`, so a failed build skips both, while a target that merely skipped, being disabled or unchanged, still lets the release and the Docker push proceed, and a package target's separate publish job `needs:` the release-task call for the same reason (D4.5). A first `plan` job decides once whether the run publishes, admitting a code-affecting bot push to `main`, a dispatch of `main` or `develop`, and a `main`-only schedule, and every publishing job gates on that decision (D4.1). - **D5:** each cross-job transfer artifact has a delete step at its consumer, gated so it runs exactly when the consumption happened rather than when the whole job succeeded (D5.2), `continue-on-error: true`, tolerating a failed listing, and looping all ids; **every** upload sets `retention-days: 1`; **no** cleanup step enumerates and deletes the run's whole artifact set, in whatever jq or API shape it is written. - **D6:** the release download uses `pattern:`/`merge-multiple:` (no `artifact-ids:`). Branch-derived config reads `inputs.branch` (a `github.ref_name` in such config is a finding). Artifact names are branch-suffixed. The target set is consistent across every surface D6.4 names: the `enable_` input, the `build-` job, that job's entries in the `github-release` and `build-docker` `needs:` lists, the `changes` paths-filter entry **and** its output, the `smoke-build` enable-forward, and any separate `publish-` job the package-registry seam requires. The `inputs.branch` rule above binds a called leaf, while a `publish-` job is in the publisher and reads `github.ref_name` correctly. -- **D7:** the publisher concurrency group is global and ref-independent with `cancel-in-progress: false`. A reusable job declares `permissions:` only where every caller grants that scope at startup, per D7.2. A boolean used by both `workflow_call` and `workflow_dispatch` is declared in both trigger blocks and compared against `true` and `'true'` (D7.3). Every cross-job condition that admits a skipped dependency pairs its allowlist with a status-check function such as `always()` or `!failure() && !cancelled()`, since the implicit `success()` is false the moment a `needs:` job skipped (D7.4). +- **D7:** the publisher concurrency group is global and ref-independent with `cancel-in-progress: false`. A reusable job declares `permissions:` only where every caller grants that scope at startup, per D7.2. A boolean used by both `workflow_call` and `workflow_dispatch` is declared in both trigger blocks, and a boolean read through `github.event.inputs.` is compared against `'true'` alone, a comparison against the boolean `true` never firing on a string, while an `inputs.` read carries the declared boolean and is used directly, a both-forms comparison there being redundant rather than a finding, so a repo whose booleans all arrive by `workflow_call` records the comparison half N/A. A job or step output is a string too and takes the same `== 'true'` (D7.3). Every cross-job condition that admits a skipped dependency pairs its allowlist with a status-check function such as `always()` or `!failure() && !cancelled()`, since the implicit `success()` is false the moment a `needs:` job skipped (D7.4). - **D8/D9:** the merge-bot enables auto-merge on `opened`/`reopened` for every Dependabot tier including semver-major, dispatches `--squash`/`--merge` by the PR's base ref, disables on a maintainer-pushed `synchronize`, and keys concurrency on the PR number rather than `github.ref`. Codegen runs as a matrix over both branches, and Dependabot targets both branches with security PRs to the default branch (D8.2). The upstream tracker's `bump-branch-prefix` and `branches` match a merge-bot rule, unless it sets `auto-merge: false`, which prefixes the head so no rule matches (wrapper repos). A gate comparing `github.actor` against hard-coded bot identities emits a `::warning::` on the non-matching branch, and the annotation is optional only where the failure announces itself anyway (D8.4). Actions are SHA-pinned. Names/shells/conditionals follow section 2, and line endings follow `.editorconfig` (D9.5), except workflow YAML, which section 2 fixes at LF. **Per-type addenda (apply only the ones present):** diff --git a/docs/reusable-workflows.md b/docs/reusable-workflows.md index 971a95ed..d65b188a 100644 --- a/docs/reusable-workflows.md +++ b/docs/reusable-workflows.md @@ -156,7 +156,7 @@ Hub: `validate-task.yml` hosts a `lint` job (the fleet doc-lint block, language - [x] Hook override path observed on a hub pull request run, [proof run][override-path-run] (runs `./.github/actions/validate`, no hub checkout). Default path observed on PhotoCleaner's adoption pull request, [pilot smoke run][pilot-smoke-run], where the hub's `validate-default` ran because that repo carries no `validate` hook. The follow-up self-reference pilot also runs the bundled prose and repository gates through `$/.github/actions/` without checking out the hub. - [x] PhotoCleaner (pilot, release trigger shape with smoke, the same repo that piloted stage 1): ptr727/PhotoCleaner#55 on `develop` (`c80cb29`), promoted in ptr727/PhotoCleaner#56 (`fa91db0`), both on 2026-08-16. `test-pull-request.yml` calls the hub validate task and no repo hook was needed. - [ ] HomeAutomation-Config (second pilot, operational trigger shape) -- [ ] The remaining repos, one checkbox each added when the pilots close, since the sweep list is every cataloged repo. A Python adopter owes one precondition before its bump: the `unit-test` job fails when the run wrote no root `coverage.xml`, and that job's Python leg runs where the repository root carries `pyproject.toml`, `tests/`, and `uv.lock`, so an adopter of that shape puts `pytest-cov` in a dev group and a `--cov=` selector in its own `pyproject.toml` before it bumps, per D1.6, which binds that selector for every Python repo with tests, lint-only excepted. aiopurpleair and Financial-Modeling carry both, homeassistant-purpleair and PlexCleaner carry no `uv.lock` so the step never runs there, and ESPHome-Config's Python is lint-only. The hub cannot smoke-test this itself, having no `tests/` and no `uv.lock` of its own. +- [ ] The remaining repos, one checkbox each added when the pilots close, since the sweep list is every cataloged repo. A Python adopter owes one precondition before its bump: the `unit-test` job fails when the run wrote no root `coverage.xml`, and that job's Python leg runs where the repository root carries `pyproject.toml`, `tests/`, and a dependency manifest it installs from, a committed `uv.lock` or a root `requirements*.txt`, so an adopter of that shape puts `pytest-cov` among its test dependencies and a `--cov=` selector in its own `pyproject.toml` before it bumps, per D1.6, which binds that selector for every Python repo with tests, lint-only excepted. aiopurpleair and Financial-Modeling carry both. homeassistant-purpleair owes the same precondition, its `requirements*.txt` and `tests/` reaching the leg though it carries no `uv.lock`, and its pytest run is a matrix over three Home Assistant versions that the hub validator does not express, so its adoption is a design question rather than a bump. PlexCleaner's Python is a stdlib-only tooling subtree with no tests, and ESPHome-Config's Python is lint-only. The hub cannot smoke-test this itself, having no `tests/` of its own. - [ ] `reports/workflow-reuse.md` regenerated with `validate-task.yml` at 0 copies (a hub-only file no repo carries) and `test-pull-request.yml` showing callers equal to copies. ### Stage 3: The Pure Functions diff --git a/reports/canonical-review.json b/reports/canonical-review.json index fef96204..8000ab01 100644 --- a/reports/canonical-review.json +++ b/reports/canonical-review.json @@ -459,19 +459,19 @@ }, { "unit": "WORKFLOW.md > 4. Behavioral Contract: Expected Outcomes", - "digest": "sha256:a4b85d3b43f98646b6cf1a7161acacc09f3e4b8f3c23c1197af69d13bba74d84", + "digest": "sha256:3616be733206b40f62b360374e08056785038ec7b528c7746ab0a5dfaffc34c3", "reviewer": "agent-skill", - "findings": 0, - "hubCommit": "4fdc718663d0149994e496e890c20c7a77f27c96", - "stamp": "2026-09-03T03:04:24Z" + "findings": 16, + "hubCommit": "28dafdf46ee35cc9901e845a317991d909a94fac", + "stamp": "2026-09-03T18:21:27Z" }, { "unit": "WORKFLOW.md > 5. Test Methodology", - "digest": "sha256:18e41e5b607eb337226934496038656cdb047c0cfb99e6bce5c8f7832dfc74b4", + "digest": "sha256:e4c043871a8fe90f89741b36893a0a4b8cf8ddba18ca8259a4513a300931ed24", "reviewer": "agent-skill", - "findings": 38, - "hubCommit": "6525cb888406c7f6a1f43c5f9b21fb041200c601", - "stamp": "2026-09-03T17:41:49Z" + "findings": 12, + "hubCommit": "28dafdf46ee35cc9901e845a317991d909a94fac", + "stamp": "2026-09-03T18:21:33Z" }, { "unit": "WORKFLOW.md > 6. Per-Project-Type Test Walkthroughs", diff --git a/scripts/tests/test_release_guards.py b/scripts/tests/test_release_guards.py index d5c79317..ee6d51e2 100755 --- a/scripts/tests/test_release_guards.py +++ b/scripts/tests/test_release_guards.py @@ -3,6 +3,7 @@ from __future__ import annotations +import re import unittest from pathlib import Path from subprocess import run @@ -10,6 +11,63 @@ REPO = Path(__file__).resolve().parents[2] +def hash_files(pattern: str, present: set[str]) -> bool: + """Whether a workflow `hashFiles()` would match anything in `present`. + + `**` spans directory separators and `*` does not, which is what separates a root-only + `requirements*.txt` from a recursive `tests/**`. + """ + regex = re.escape(pattern).replace(r"\*\*", "@@").replace(r"\*", "[^/]*").replace("@@", ".*") + return any(re.fullmatch(regex, path) for path in present) + + +def split_top_level(expression: str, operator: str) -> list[str]: + """Split on `operator` outside any parentheses.""" + parts: list[str] = [] + depth = 0 + start = 0 + index = 0 + while index < len(expression): + character = expression[index] + if character == "(": + depth += 1 + elif character == ")": + depth -= 1 + elif depth == 0 and expression.startswith(operator, index): + parts.append(expression[start:index]) + index += len(operator) + start = index + continue + index += 1 + parts.append(expression[start:]) + return [part.strip() for part in parts] + + +def evaluate_guard(expression: str, present: set[str]) -> bool: + """Evaluate a workflow `if:` written only from `hashFiles(...)` emptiness tests, `&&`, `||`, `()`. + + Deliberately narrow rather than a general expression engine: it is here to answer what the + validator's Python leg does for one file set, not to reimplement GitHub's evaluator. + """ + + def atom(text: str) -> bool: + match = re.fullmatch(r"hashFiles\('([^']*)'\)\s*(!=|==)\s*''", text.strip()) + if not match: + raise ValueError(f"unsupported guard atom: {text!r}") + hit = hash_files(match.group(1), present) + return hit if match.group(2) == "!=" else not hit + + result = True + for clause in split_top_level(expression, "&&"): + if clause.startswith("(") and clause.endswith(")"): + result = result and any( + atom(alternative) for alternative in split_top_level(clause[1:-1], "||") + ) + else: + result = result and atom(clause) + return result + + class ReleaseGuardCase(unittest.TestCase): """Publishing and audit discovery require their prerequisite checks to succeed.""" @@ -158,6 +216,43 @@ def test_audit_probes_fail_before_local_path_checks(self) -> None: audit, ) + def test_validator_python_leg_reaches_a_pip_dependency_repo(self) -> None: + """WORKFLOW.md D1.6 owes coverage to every Python repo with tests, uv-managed or not. + + Gating the leg on `uv.lock` alone skipped a pip/requirements repo that has tests, so it + collected no coverage and never reached the missing-report failure either. + """ + workflow = (REPO / ".github/workflows/validate-task.yml").read_text(encoding="utf-8") + job = workflow.split("\n unit-test:\n", 1)[1].split("\n validate:\n", 1)[0] + guards = [ + " ".join(line.strip() for line in block.strip().splitlines()) + for block in re.findall(r"(?m)^ if: >-\n((?:^ {10}.*\n)+)", job) + ] + python_guards = [guard for guard in guards if "tests/**" in guard] + + # Setup, dependency install, pytest, and upload: one drifting guard reintroduces the skip. + self.assertEqual(4, len(python_guards)) + self.assertEqual(1, len(set(python_guards))) + + trees = { + "uv project with tests": ({"pyproject.toml", "uv.lock", "tests/test_a.py"}, True), + "pip project with tests": ( + {"pyproject.toml", "requirements.txt", "requirements-test.txt", "tests/test_a.py"}, + True, + ), + "tests but no dependency manifest": ({"pyproject.toml", "tests/test_a.py"}, False), + "lint-only scripts tree": ({"pyproject.toml", "scripts/tool.py"}, False), + "pip project with no tests": ({"pyproject.toml", "requirements.txt"}, False), + } + for label, (present, expected) in trees.items(): + with self.subTest(tree=label): + self.assertEqual(expected, evaluate_guard(python_guards[0], present)) + + # The guard admitting a pip repo is only half of it: the steps must install and run without a lockfile. + self.assertIn("uv pip install -r", job) + self.assertIn(".venv/bin/python -m pytest --cov-report=xml", job) + self.assertEqual(2, job.count("if [ -f uv.lock ]; then")) + def test_audit_bash_blocks_are_not_labeled_as_posix_shell(self) -> None: audit_lines = (REPO / "AUDIT.md").read_text(encoding="utf-8").splitlines() bash_only = ("<(", "<<<", "$'", "[[") diff --git a/spec/audit.py b/spec/audit.py index e715f47b..fb36fd82 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -197,6 +197,42 @@ def repo_tree(slug, ground_head): return None if entries is None else set(entries) +def coverage_claiming_types(types, repo_profiles, type_mechanisms, tree): + """The declared types that owe Codecov coverage, meaning the CODECOV_TOKEN secret and codecov.yml. + + A type owes coverage when the fleet maps it to the codecov mechanism, its declared profile is not + lint-only, and the repo carries tests for it. The last condition is what keeps a package-only build + repo, a library whose tests live elsewhere or are not yet written, from being told to store a token + and commit a codecov.yml whose statuses gate a report its pipeline never produces. + + Each detector is the test-presence half of the hub validator's own guard, so the audit reads the + evidence the mechanism reads rather than a second definition free to drift from it: + `.github/workflows/validate-task.yml` keys its C# leg on a `*Tests*.csproj` anywhere in the tree and + its Python leg on a root `tests/` directory. The rest of that guard, the language and dependency + markers beside it, is deliberately not read here, since D1.6 owes coverage to every repo with tests + and a repo the hub leg does not reach meets it through its own workflows instead. + + `tree` is the repo's blob path set, or None when it could not be read in full. On None the test + question is unanswerable, so every candidate type keeps its claim: dropping a coverage requirement + over an unreadable tree would report a repo clean for a reason nobody measured. A type mapped to + codecov with no detector here keeps its claim for the same reason. + """ + detectors = { + "csharp": lambda paths: any( + p.endswith(".csproj") and "Tests" in p.rsplit("/", 1)[-1] for p in paths + ), + "python": lambda paths: any(p.startswith("tests/") for p in paths), + } + claiming = [] + for name in types: + if type_mechanisms.get(name) != "codecov" or repo_profiles.get(name) == "lint-only": + continue + detect = detectors.get(name) + if tree is None or detect is None or detect(tree): + claiming.append(name) + return claiming + + @functools.cache def canonical_blob_sha(path): """The hub's git blob identity for path, from the same resolved `main` commit @@ -2237,16 +2273,20 @@ def audit_repo(entry, spec, branch=None): # --- Secrets (names only) --- secrets = spec["secrets"] - # The codecov coverage requirement, meaning the CODECOV_TOKEN secret and the codecov.yml file, is claimed by a type only at build profile. - # A lint-only language has no tests and so no coverage, per spec/type-model.md. + # The codecov coverage requirement, meaning the CODECOV_TOKEN secret and the codecov.yml file, is claimed by a type only at build profile and only where the repo carries tests for that type. + # A lint-only language has no tests and so no coverage, per spec/type-model.md, and a build-profile language with no test suite has none either: the validator's leg never runs, so the token and the codecov.yml would gate on a report the pipeline cannot produce. + # The tree is read here rather than at the verbatim-tree section below so one call answers both, and the finding for an unreadable tree stays where it was. repo_profiles = entry.get("profiles", {}) if not isinstance(repo_profiles, dict): repo_profiles = {} - coverage_active = any( - secrets.get("typeMechanisms", {}).get(t) == "codecov" - and repo_profiles.get(t) != "lint-only" - for t in types + carried_entries = repo_tree_entries(slug, ground_head) + coverage_types = coverage_claiming_types( + types, + repo_profiles, + secrets.get("typeMechanisms", {}), + None if carried_entries is None else set(carried_entries), ) + coverage_active = bool(coverage_types) stores = {} # There is no ok404 here, since an empty store returns {"secrets": []}, so a 404 or 403 from permissions or a rename must surface as ERROR rather than cascade into false missing-secret DEFECTs. for store, path in [ @@ -2258,11 +2298,15 @@ def audit_repo(entry, spec, branch=None): mechanisms = [ secrets["targetMechanisms"].get(p.get("target")) for p in entry.get("publish", []) ] - mechanisms += [ - secrets.get("typeMechanisms", {}).get(t) - for t in types - if repo_profiles.get(t) != "lint-only" - ] + # The codecov mechanism is claimed from coverage_types above rather than from the profile alone, so a + # build-profile language with no tests requires no CODECOV_TOKEN, matching the codecov.yml skip below. + for name in types: + if repo_profiles.get(name) == "lint-only": + continue + mechanism = secrets.get("typeMechanisms", {}).get(name) + if mechanism == "codecov" and name not in coverage_types: + continue + mechanisms.append(mechanism) claimed = [secrets["mechanisms"][m] for m in mechanisms if m and m in secrets["mechanisms"]] required_by_store = {"actions": set(), "dependabot": set()} for store in secrets["baseline"].get("stores", []): @@ -2491,7 +2535,7 @@ def audit_repo(entry, spec, branch=None): ) # --- Manifest-owned verbatim trees --- - carried_entries = repo_tree_entries(slug, ground_head) + # carried_entries was read once above, where the coverage applicability check needed it first. if carried_entries is None: findings.append( ( @@ -5142,6 +5186,38 @@ def _selftest(): " ok repo_tree: a truncated tree and a missing tree sha both return None, and a whole one drops non-blobs" ) + # Coverage applicability turns on tests rather than on the profile alone, so a package-only build repo + # is N/A instead of being told to store a token for a report its pipeline never produces. + type_mechs = {"csharp": "codecov", "python": "codecov"} + coverage_cases = [ + (["python"], {}, {"src/pkg/__init__.py", "pyproject.toml"}, [], "python, no tests"), + (["python"], {}, {"tests/test_a.py", "pyproject.toml"}, ["python"], "python with tests"), + (["python"], {"python": "lint-only"}, {"tests/test_a.py"}, [], "python lint-only"), + (["csharp"], {}, {"src/App/App.csproj"}, [], "csharp, no test project"), + (["csharp"], {}, {"test/App.Tests/App.Tests.csproj"}, ["csharp"], "csharp with tests"), + (["csharp"], {}, {"src/Tests/App.csproj"}, [], "csharp, Tests is a directory name only"), + ( + ["csharp", "python"], + {"python": "lint-only"}, + {"test/App.Tests/App.Tests.csproj", "tools/py/pyproject.toml"}, + ["csharp"], + "mixed repo, the tested C# side claims coverage", + ), + (["python"], {}, None, ["python"], "unreadable tree keeps the claim"), + (["docker"], {}, {"tests/test_a.py"}, [], "a type mapped to no mechanism claims nothing"), + ] + coverage_ok = True + for types_in, profiles_in, tree_in, expected, label in coverage_cases: + got = coverage_claiming_types(types_in, profiles_in, type_mechs, tree_in) + if got != expected: + ok = False + coverage_ok = False + print(f" FAIL coverage_claiming_types [{label}] -> {got}, expected {expected}") + if coverage_ok: + print( + " ok coverage_claiming_types: tests decide the claim, lint-only never claims, and an unreadable tree keeps it" + ) + id_cases = [ ("https://github.com/owner/Repo", "owner/repo"), ("https://github.com/owner/Repo/", "owner/repo"), diff --git a/spec/project-types.json b/spec/project-types.json index f329ac22..3301a965 100644 --- a/spec/project-types.json +++ b/spec/project-types.json @@ -42,7 +42,7 @@ { "id": "python.pyright.config", "verdict": "intent", "assert": "Build profile: pyright is configured and runs strict on first-party code (src or the integration package) - the strong typing baseline. Third-party strictness is relaxed only where a dependency has no usable types. N/A for the lint-only profile, whose type checker is mypy over stdlib-only code (python.mypy.allowed).", "intentRef": "CODESTYLE.md", "minProfile": "build" }, { "id": "python.config.placement", "verdict": "letter", "assert": "ruff and the type-checker config live in pyproject.toml (canonical); standalone .ruff.toml / pyrightconfig.json is a drift finding. A Home Assistant integration is the exception - it follows home-assistant/core standalone-config conventions and is scored by ha.python.conventions instead.", "intentRef": "CODESTYLE.md" }, { "id": "python.mypy.allowed", "verdict": "intent", "assert": "mypy is permitted as an additional type checker, not banned. It is required for a Home Assistant integration (platinum strict-typing) and is the lint-only profile's type checker. When used it runs in CI and the editor.", "intentRef": "CODESTYLE.md" }, - { "id": "python.coverage.codecov", "verdict": "letter", "assert": "The test job collects coverage (pytest --cov-report=xml) and uploads it to Codecov via codecov/codecov-action, best-effort (continue-on-error and fail_ci_if_error: false). Because --cov-report=xml alone measures nothing and writes no file, the repo declares pytest-cov in a dev dependency group and selects the coverage source in its own pyproject.toml, an addopts --cov= entry in practice, leaving the report at the repo root as coverage.xml. The validator's Python leg, which runs where the repository root carries pyproject.toml, tests/ and uv.lock, deletes any coverage.xml before the run and fails the test step when that file was not written, so a repo of that shape missing the selector reds its gate there, while one missing pytest-cov reds it earlier, at the pytest invocation that does not recognize the flag. CODECOV_TOKEN is stored in both the repo actions and dependabot secrets stores, the second so the upload does not skip on a Dependabot PR, and the caller maps it to the reusable validator by name. Required for every Python repo with tests. N/A for the lint-only profile (its unittest suite runs under coverage in CI, reported without a threshold and never uploaded to Codecov). In a mixed repo the codecov.yml file-presence is still required by any co-present type that has tests, e.g. csharp.", "intentRef": "WORKFLOW.md", "minProfile": "build" }, + { "id": "python.coverage.codecov", "verdict": "letter", "assert": "The test job collects coverage (pytest --cov-report=xml) and uploads it to Codecov via codecov/codecov-action, best-effort (continue-on-error and fail_ci_if_error: false). Because --cov-report=xml alone measures nothing and writes no file, the repo declares pytest-cov among its test dependencies, a dev dependency group in a uv project and a requirements*.txt entry on pip, and selects the coverage source in its own pyproject.toml, an addopts --cov= entry in practice, leaving the report at the repo root as coverage.xml. The validator's Python leg, which runs where the repository root carries pyproject.toml, tests/ and a dependency manifest it can install from, being a committed uv.lock or a root requirements*.txt, deletes any coverage.xml before the run and fails the test step when that file was not written, so a repo of that shape missing the selector reds its gate there, while one missing pytest-cov reds it earlier, at the pytest invocation that does not recognize the flag. CODECOV_TOKEN is stored in both the repo actions and dependabot secrets stores, the second so the upload does not skip on a Dependabot PR, and the caller maps it to the reusable validator by name. Required for every Python repo with tests, whatever its dependency mechanism. N/A for a repo carrying no tests for this type, whose validator leg never runs, and N/A for the lint-only profile (its unittest suite runs under coverage in CI, reported without a threshold and never uploaded to Codecov). In a mixed repo the codecov.yml file-presence is still required by any co-present type that has tests, e.g. csharp.", "intentRef": "WORKFLOW.md", "minProfile": "build" }, { "id": "python.uvlock.pinned", "verdict": "letter", "assert": "Build profile: the committed uv.lock resolves to LF through the repository-wide .editorconfig and .gitattributes defaults. A CRLF-native operational repo adds a narrow uv.lock LF override only if it adopts the uv build profile. N/A for a non-uv Python repo (a Home Assistant integration on pip/requirements) and for the lint-only profile (no uv.lock by definition).", "intentRef": "GOVERNANCE.md#line-endings", "minProfile": "build" }, { "id": "python.scripts.uvx", "verdict": "letter", "assert": "Lint-only profile only: the tools run via uvx (no project install, no lockfile). A uvx @ pin in a run: step is not Dependabot-trackable, so CI runs uvx ruff@latest / uvx mypy@latest - the fleet rule pins only what Dependabot auto-updates and otherwise runs latest, never a manual pin that goes stale. VS Code tasks, README, and CI all run the unpinned latest. N/A for the build profile (which pins tool versions via uv.lock + uv sync --frozen instead).", "intentRef": "CODESTYLE.md" } ] diff --git a/spec/type-model.md b/spec/type-model.md index 40a610ec..ebea3a57 100644 --- a/spec/type-model.md +++ b/spec/type-model.md @@ -34,13 +34,13 @@ A language type is present at one of two **depths**, declared as its `profile`: - **build** - the language is compiled, tested, and/or packaged in this repo. Its full check set applies (style, type-check, tests, coverage, packaging). - **lint-only** - the language is present and style-checked here, but not built: there is no build/test/package for it in this repo. Only its lint/style/type-check checks apply. Build, test, coverage, and packaging checks are N/A. -Each check may declare the **minimum profile** it needs via a `minProfile` field. A check without one applies at every profile, and a check with `minProfile: build` applies only at `build`. So lint/style/type-check checks omit it, while build/test/coverage/package checks set `build`. The audit uses the declared profile to hold the coverage requirement (the CODECOV_TOKEN secret and the codecov.yml file) as N/A for a lint-only language, replacing the older per-check "N/A for the SCRIPTS profile" prose. +Each check may declare the **minimum profile** it needs via a `minProfile` field. A check without one applies at every profile, and a check with `minProfile: build` applies only at `build`. So lint/style/type-check checks omit it, while build/test/coverage/package checks set `build`. The audit uses the declared profile to hold the coverage requirement (the CODECOV_TOKEN secret and the codecov.yml file) as N/A for a lint-only language, replacing the older per-check "N/A for the SCRIPTS profile" prose. The profile is a floor rather than the whole test, since a build-profile language can still carry no test suite, so the audit reads the repo's tree for tests as well and holds the same requirement N/A where it finds none. The profile is **declared and validated**, not merely detected. `python` already reads its shape structurally from `pyproject.toml` (a uv PROJECT with tests and a lockfile, versus stdlib SCRIPTS tooling). That structural read becomes the profile **validator**. A declared `python` profile that contradicts the pyproject shape is a false declaration. One concept (the declared profile), checked by detection, rather than two ways to classify. ### Consequence for cross-cutting checks -A cross-cutting check that presumes a built, tested language must respect the profile. In particular the coverage checks (the `CODECOV_TOKEN` secret and the `codecov.yml` file presence) are **profile-aware**: they are N/A for a language whose declared profile has no tests. A lint-only language must never manufacture a coverage finding. +A cross-cutting check that presumes a built, tested language must respect the profile. In particular the coverage checks (the `CODECOV_TOKEN` secret and the `codecov.yml` file presence) are **profile-aware and tests-aware**: they are N/A for a language whose declared profile has no tests, and N/A for a build-profile language the repo carries no tests for. A lint-only language must never manufacture a coverage finding, and neither must a package-only build language whose tests live elsewhere or are not yet written. ## Languages From 382c0185dda69811a190d17ee305565589aa2ec8 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 3 Sep 2026 11:39:50 -0700 Subject: [PATCH 2/4] Install the Project on the Pip Branch and Resolve Requirements Together The local review found the pip branch never installed the project under test, so a repo with a PEP 621 [project] table and a src layout failed collection with ModuleNotFoundError where the lockfile branch's uv sync had installed it. The branch now installs it editable wherever pyproject declares that table, and a repo declaring none, a Home Assistant custom_components layout, is left alone. Both shapes were run end to end before and after. It also installed each requirements file in its own invocation, so the files were never resolved together, and the glob sorts the base file last, letting a base pin downgrade what the test-requirements file had just resolved with no conflict reported. The matches are collected and resolved in one invocation. The audit's test detectors were the validator's guards exactly, so a repo whose suite sits in test/ or whose project is named App.UnitTest.csproj read as having none and lost its coverage claim silently. They are now deliberately broader, since a claim kept wrongly is a finding a reader dismisses and a claim dropped wrongly is a clean report on a repo nobody measured. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/validate-task.yml | 15 ++++++++-- docs/reusable-workflows.md | 2 +- scripts/tests/test_release_guards.py | 12 +++++++- spec/audit.py | 42 ++++++++++++++++++++-------- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 8b61da20..dc0c6f6f 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -259,7 +259,7 @@ jobs: # Every step below carries its own tree-detection guard instead. # A caller with neither a *Tests*.csproj nor a tests/ directory beside a pyproject.toml and a dependency manifest runs every step's guard false, and the job reports success having done nothing, which is the clean skip this job promises. # That manifest is a committed uv.lock or a root requirements*.txt, the two dependency mechanisms spec/project-types.json names, so a pip-based Python repo with tests is served here rather than skipped, per D1.6. - # The lint-only Python profile (spec/project-types.json python profileNote) carries neither, being stdlib-only and uvx-run, so it stays excluded. + # What excludes the lint-only Python profile here is the root tests/ guard rather than the manifest one, that profile being tooling scripts embedded in a non-Python repo rather than a tested package at the root. unit-test: name: Unit test job runs-on: ubuntu-latest @@ -309,7 +309,9 @@ jobs: # The dependency mechanism is read from the tree rather than assumed: a committed uv.lock is the uv project shape and syncs frozen, and a root requirements*.txt is the pip shape a non-uv Python repo carries instead. # The pip branch installs into the .venv that uv pip install resolves by default, so the pytest step below can name that interpreter directly and needs no activation carried between steps. - # Every requirements*.txt is installed rather than one canonical name, since the test dependencies live in a second file whose spelling differs by repo. + # Every requirements*.txt is collected rather than one canonical name, since the test dependencies live in a second file whose spelling differs by repo, and the collected set is resolved in one invocation. + # Installing them one file at a time would resolve each alone, and the glob sorts the base file last, so a base pin would silently downgrade what a test-requirements file had just resolved and the break would surface inside pytest rather than at install. + # The lockfile branch installs the project itself through uv sync, so the pip branch installs it too wherever pyproject declares a [project] table, and a repo declaring none (a Home Assistant custom_components layout) is left alone rather than failing on a package it does not build. - name: Sync dependencies step if: >- hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' @@ -320,10 +322,17 @@ jobs: uv sync --all-groups --frozen else uv venv + requirement_args=() for file in requirements*.txt; do [ -e "$file" ] || continue - uv pip install -r "$file" + requirement_args+=(-r "$file") done + if [ "${#requirement_args[@]}" -gt 0 ]; then + uv pip install "${requirement_args[@]}" + fi + if grep -Eq '^[[:space:]]*\[project\]' pyproject.toml; then + uv pip install -e . + fi fi # --cov-report=xml names the report format and selects nothing to measure, so the repository's own pyproject.toml supplies the --cov selector, per D1.6. diff --git a/docs/reusable-workflows.md b/docs/reusable-workflows.md index d65b188a..e2cfdc61 100644 --- a/docs/reusable-workflows.md +++ b/docs/reusable-workflows.md @@ -156,7 +156,7 @@ Hub: `validate-task.yml` hosts a `lint` job (the fleet doc-lint block, language - [x] Hook override path observed on a hub pull request run, [proof run][override-path-run] (runs `./.github/actions/validate`, no hub checkout). Default path observed on PhotoCleaner's adoption pull request, [pilot smoke run][pilot-smoke-run], where the hub's `validate-default` ran because that repo carries no `validate` hook. The follow-up self-reference pilot also runs the bundled prose and repository gates through `$/.github/actions/` without checking out the hub. - [x] PhotoCleaner (pilot, release trigger shape with smoke, the same repo that piloted stage 1): ptr727/PhotoCleaner#55 on `develop` (`c80cb29`), promoted in ptr727/PhotoCleaner#56 (`fa91db0`), both on 2026-08-16. `test-pull-request.yml` calls the hub validate task and no repo hook was needed. - [ ] HomeAutomation-Config (second pilot, operational trigger shape) -- [ ] The remaining repos, one checkbox each added when the pilots close, since the sweep list is every cataloged repo. A Python adopter owes one precondition before its bump: the `unit-test` job fails when the run wrote no root `coverage.xml`, and that job's Python leg runs where the repository root carries `pyproject.toml`, `tests/`, and a dependency manifest it installs from, a committed `uv.lock` or a root `requirements*.txt`, so an adopter of that shape puts `pytest-cov` among its test dependencies and a `--cov=` selector in its own `pyproject.toml` before it bumps, per D1.6, which binds that selector for every Python repo with tests, lint-only excepted. aiopurpleair and Financial-Modeling carry both. homeassistant-purpleair owes the same precondition, its `requirements*.txt` and `tests/` reaching the leg though it carries no `uv.lock`, and its pytest run is a matrix over three Home Assistant versions that the hub validator does not express, so its adoption is a design question rather than a bump. PlexCleaner's Python is a stdlib-only tooling subtree with no tests, and ESPHome-Config's Python is lint-only. The hub cannot smoke-test this itself, having no `tests/` of its own. +- [ ] The remaining repos, one checkbox each added when the pilots close, since the sweep list is every cataloged repo. A Python adopter owes one precondition before its bump: the `unit-test` job fails when the run wrote no root `coverage.xml`, and that job's Python leg runs where the repository root carries `pyproject.toml`, `tests/`, and a dependency manifest it installs from, a committed `uv.lock` or a root `requirements*.txt`, so an adopter of that shape puts `pytest-cov` among its test dependencies and a `--cov=` selector in its own `pyproject.toml` before it bumps, per D1.6, which binds that selector for every Python repo with tests, lint-only excepted. aiopurpleair and Financial-Modeling carry both. homeassistant-purpleair owes the same precondition, its `requirements*.txt` and `tests/` reaching the leg though it carries no `uv.lock`, and its adoption is a design question rather than a bump, its pytest run being a matrix over several Home Assistant versions on Python 3.14 where the hub leg pins 3.13 and expresses no matrix at all. PlexCleaner's Python is a stdlib-only tooling subtree with no tests, and ESPHome-Config's Python is lint-only. The hub cannot smoke-test this itself, having no `tests/` of its own. - [ ] `reports/workflow-reuse.md` regenerated with `validate-task.yml` at 0 copies (a hub-only file no repo carries) and `test-pull-request.yml` showing callers equal to copies. ### Stage 3: The Pure Functions diff --git a/scripts/tests/test_release_guards.py b/scripts/tests/test_release_guards.py index ee6d51e2..3cad6aa8 100755 --- a/scripts/tests/test_release_guards.py +++ b/scripts/tests/test_release_guards.py @@ -249,10 +249,20 @@ def test_validator_python_leg_reaches_a_pip_dependency_repo(self) -> None: self.assertEqual(expected, evaluate_guard(python_guards[0], present)) # The guard admitting a pip repo is only half of it: the steps must install and run without a lockfile. - self.assertIn("uv pip install -r", job) + self.assertIn('requirement_args+=(-r "$file")', job) + self.assertIn('uv pip install "${requirement_args[@]}"', job) self.assertIn(".venv/bin/python -m pytest --cov-report=xml", job) self.assertEqual(2, job.count("if [ -f uv.lock ]; then")) + # One resolve over every requirements file, never one install per file. + # The glob sorts the base file last, so a per-file install lets its pins downgrade what the test-requirements file just resolved. + self.assertNotIn('uv pip install -r "$file"', job) + + # The lockfile branch installs the project itself, so the pip branch owes the same. + # Without it a src-layout repo fails collection on its own package instead of running its tests. + self.assertIn("uv pip install -e .", job) + self.assertIn(r"grep -Eq '^[[:space:]]*\[project\]' pyproject.toml", job) + def test_audit_bash_blocks_are_not_labeled_as_posix_shell(self) -> None: audit_lines = (REPO / "AUDIT.md").read_text(encoding="utf-8").splitlines() bash_only = ("<(", "<<<", "$'", "[[") diff --git a/spec/audit.py b/spec/audit.py index fb36fd82..3cb88288 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -205,23 +205,25 @@ def coverage_claiming_types(types, repo_profiles, type_mechanisms, tree): repo, a library whose tests live elsewhere or are not yet written, from being told to store a token and commit a codecov.yml whose statuses gate a report its pipeline never produces. - Each detector is the test-presence half of the hub validator's own guard, so the audit reads the - evidence the mechanism reads rather than a second definition free to drift from it: - `.github/workflows/validate-task.yml` keys its C# leg on a `*Tests*.csproj` anywhere in the tree and - its Python leg on a root `tests/` directory. The rest of that guard, the language and dependency - markers beside it, is deliberately not read here, since D1.6 owes coverage to every repo with tests - and a repo the hub leg does not reach meets it through its own workflows instead. + Each detector is deliberately broader than the hub validator's own guard, which keys its C# leg on + `**/*Tests*.csproj` and its Python leg on a root `tests/` directory. D1.6 owes coverage to every repo + with tests, and a repo the hub leg does not reach meets it through its own workflows instead, so a + detector matching the guard exactly would drop the claim for a repo whose suite sits in `test/` or + whose project is named `App.UnitTest.csproj`. Every uncertainty here therefore resolves toward + keeping the claim, since a claim kept wrongly surfaces as a finding a reader can dismiss and a claim + dropped wrongly is a silent clean report on a repo nobody measured. `tree` is the repo's blob path set, or None when it could not be read in full. On None the test - question is unanswerable, so every candidate type keeps its claim: dropping a coverage requirement - over an unreadable tree would report a repo clean for a reason nobody measured. A type mapped to - codecov with no detector here keeps its claim for the same reason. + question is unanswerable, so every candidate type keeps its claim, for that same reason. A type + mapped to codecov with no detector here keeps its claim too. """ detectors = { "csharp": lambda paths: any( - p.endswith(".csproj") and "Tests" in p.rsplit("/", 1)[-1] for p in paths + p.endswith(".csproj") and "Test" in p.rsplit("/", 1)[-1] for p in paths + ), + "python": lambda paths: any( + segment in ("test", "tests") for p in paths for segment in p.split("/")[:-1] ), - "python": lambda paths: any(p.startswith("tests/") for p in paths), } claiming = [] for name in types: @@ -2385,7 +2387,7 @@ def audit_repo(entry, spec, branch=None): continue path = item["path"] if path == "codecov.yml" and not coverage_active: - continue # coverage feature file: N/A when no type claims codecov at build profile (spec/type-model.md) + continue # coverage feature file: N/A when no type claims codecov at build profile with tests present (spec/type-model.md) if path not in wanted_sections: wanted_sections[path] = set() verbatim_secs[path] = set() @@ -5196,6 +5198,22 @@ def _selftest(): (["csharp"], {}, {"src/App/App.csproj"}, [], "csharp, no test project"), (["csharp"], {}, {"test/App.Tests/App.Tests.csproj"}, ["csharp"], "csharp with tests"), (["csharp"], {}, {"src/Tests/App.csproj"}, [], "csharp, Tests is a directory name only"), + ( + ["csharp"], + {}, + {"test/App.UnitTest.csproj"}, + ["csharp"], + "csharp, singular Test in the name", + ), + ( + ["python"], + {}, + {"test/test_a.py", "pyproject.toml"}, + ["python"], + "python, singular test dir", + ), + (["python"], {}, {"src/tests/test_a.py"}, ["python"], "python, tests below the root"), + (["python"], {}, {"tests"}, [], "python, a file named tests is not a directory"), ( ["csharp", "python"], {"python": "lint-only"}, From 879262f70114aaf816f888e0f98a06b6cec8b880 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 3 Sep 2026 11:54:27 -0700 Subject: [PATCH 3/4] Widen the Audit Test Detectors and Trim the Comments They Grew The detectors read "has tests" from the file tree, and both languages had a gap that dropped a repo's coverage claim silently. A Python suite of root-level test_app.py or app_test.py modules carries no tests/ directory, and a C# project named Specs.csproj under test/ carries no Test in its name. Either read as testless, which suppressed the Codecov mechanism and held codecov.yml N/A: a clean report on a repo nobody measured, which is the direction the docstring already said to resolve against. A project neither named nor located as a test stays invisible to a tree read, since deciding it needs the project file's own contents. That residual is filed rather than guessed at. The comments added alongside grew past what the style contract allows, one line by default and a second for a constraint the code cannot carry, and two of them wrapped a sentence across lines. They come out shorter than the block that stood there before. The reusable-workflows stage summary still named uv run pytest as the one invocation, which the pip branch falsified. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/validate-task.yml | 7 +-- docs/reusable-workflows.md | 2 +- spec/audit.py | 67 +++++++++++++++++++++++++---- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index dc0c6f6f..c2210a75 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -307,11 +307,8 @@ jobs: with: python-version: "3.13" - # The dependency mechanism is read from the tree rather than assumed: a committed uv.lock is the uv project shape and syncs frozen, and a root requirements*.txt is the pip shape a non-uv Python repo carries instead. - # The pip branch installs into the .venv that uv pip install resolves by default, so the pytest step below can name that interpreter directly and needs no activation carried between steps. - # Every requirements*.txt is collected rather than one canonical name, since the test dependencies live in a second file whose spelling differs by repo, and the collected set is resolved in one invocation. - # Installing them one file at a time would resolve each alone, and the glob sorts the base file last, so a base pin would silently downgrade what a test-requirements file had just resolved and the break would surface inside pytest rather than at install. - # The lockfile branch installs the project itself through uv sync, so the pip branch installs it too wherever pyproject declares a [project] table, and a repo declaring none (a Home Assistant custom_components layout) is left alone rather than failing on a package it does not build. + # One resolve over the collected requirements files, since the glob sorts the base file last and a per-file install would let its pins downgrade what the test file just resolved. + # The editable install matches what uv sync gives the other branch, and its guard leaves a layout declaring no [project] alone. - name: Sync dependencies step if: >- hashFiles('pyproject.toml') != '' && hashFiles('tests/**') != '' diff --git a/docs/reusable-workflows.md b/docs/reusable-workflows.md index e2cfdc61..f25ddf27 100644 --- a/docs/reusable-workflows.md +++ b/docs/reusable-workflows.md @@ -148,7 +148,7 @@ Adoptable since `2.0.338`. Each repo replaces the whole of its `.github/workflow ### Stage 2: The Gates -Hub: `validate-task.yml` hosts a `lint` job (the fleet doc-lint block, language lint by tree detection, the prose gate, and the repo gate), a generic `unit-test` job (a `dotnet test` or a `uv run pytest`, skipped cleanly where the caller carries no test project), and a `validate` job resolving the `validate` hook for a repo's own domain checks, which decides #729 in the one place the `uvx` tools are pinned or floated. There is no `test-pull-request-task.yml`: the ruleset-bound aggregator stays in the caller stub by design, and a task wrapping one line that calls `validate-task.yml` hosts nothing generic, so the stub shapes live in [Adopting the Gates][adopting-the-gates] instead, with the trigger shape, operational or release, settling #585. This stage is where the hook fallback is first proven live: the hub carries its own `validate` hook (its registry and spec check, its script self-tests, its fleet-skills check, and its unclassified-character report), so a hub pull request exercises the override path, and a repo with no hook of its own exercises the default. +Hub: `validate-task.yml` hosts a `lint` job (the fleet doc-lint block, language lint by tree detection, the prose gate, and the repo gate), a generic `unit-test` job (a `dotnet test` or a `pytest`, skipped cleanly where the caller carries no test project), and a `validate` job resolving the `validate` hook for a repo's own domain checks, which decides #729 in the one place the `uvx` tools are pinned or floated. There is no `test-pull-request-task.yml`: the ruleset-bound aggregator stays in the caller stub by design, and a task wrapping one line that calls `validate-task.yml` hosts nothing generic, so the stub shapes live in [Adopting the Gates][adopting-the-gates] instead, with the trigger shape, operational or release, settling #585. This stage is where the hook fallback is first proven live: the hub carries its own `validate` hook (its registry and spec check, its script self-tests, its fleet-skills check, and its unclassified-character report), so a hub pull request exercises the override path, and a repo with no hook of its own exercises the default. - [x] Hub pull request on `develop` with the task, the hub's own hook and default, the manifest contracts, and the catalog snippets left for the release that follows, [#760][pr-760]. - [x] Promoted to `main` in #774 (`0b07a59d`) and released as `2.0.352`, the first tag carrying `validate-task.yml`. diff --git a/spec/audit.py b/spec/audit.py index 3cb88288..c41502e1 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -197,6 +197,24 @@ def repo_tree(slug, ground_head): return None if entries is None else set(entries) +def in_test_directory(path): + """Whether any directory segment of path names a test directory, compared case-insensitively.""" + return any(segment.lower() in ("test", "tests") for segment in path.split("/")[:-1]) + + +def is_python_test_module(path): + """Whether path is a module pytest discovers by name, wherever in the tree it sits.""" + name = path.rsplit("/", 1)[-1] + return name.endswith(".py") and (name.startswith("test_") or name.endswith("_test.py")) + + +def is_csharp_test_project(path): + """Whether path is a C# project either named or located as a test project.""" + return path.endswith(".csproj") and ( + "Test" in path.rsplit("/", 1)[-1] or in_test_directory(path) + ) + + def coverage_claiming_types(types, repo_profiles, type_mechanisms, tree): """The declared types that owe Codecov coverage, meaning the CODECOV_TOKEN secret and codecov.yml. @@ -218,11 +236,9 @@ def coverage_claiming_types(types, repo_profiles, type_mechanisms, tree): mapped to codecov with no detector here keeps its claim too. """ detectors = { - "csharp": lambda paths: any( - p.endswith(".csproj") and "Test" in p.rsplit("/", 1)[-1] for p in paths - ), + "csharp": lambda paths: any(is_csharp_test_project(p) for p in paths), "python": lambda paths: any( - segment in ("test", "tests") for p in paths for segment in p.split("/")[:-1] + in_test_directory(p) or is_python_test_module(p) for p in paths ), } claiming = [] @@ -2300,8 +2316,7 @@ def audit_repo(entry, spec, branch=None): mechanisms = [ secrets["targetMechanisms"].get(p.get("target")) for p in entry.get("publish", []) ] - # The codecov mechanism is claimed from coverage_types above rather than from the profile alone, so a - # build-profile language with no tests requires no CODECOV_TOKEN, matching the codecov.yml skip below. + # The codecov mechanism follows coverage_types rather than the profile, so a language with no tests requires no CODECOV_TOKEN. for name in types: if repo_profiles.get(name) == "lint-only": continue @@ -5188,8 +5203,7 @@ def _selftest(): " ok repo_tree: a truncated tree and a missing tree sha both return None, and a whole one drops non-blobs" ) - # Coverage applicability turns on tests rather than on the profile alone, so a package-only build repo - # is N/A instead of being told to store a token for a report its pipeline never produces. + # Applicability turns on tests, so a package-only build repo is N/A rather than owing a token for a report nothing produces. type_mechs = {"csharp": "codecov", "python": "codecov"} coverage_cases = [ (["python"], {}, {"src/pkg/__init__.py", "pyproject.toml"}, [], "python, no tests"), @@ -5197,7 +5211,42 @@ def _selftest(): (["python"], {"python": "lint-only"}, {"tests/test_a.py"}, [], "python lint-only"), (["csharp"], {}, {"src/App/App.csproj"}, [], "csharp, no test project"), (["csharp"], {}, {"test/App.Tests/App.Tests.csproj"}, ["csharp"], "csharp with tests"), - (["csharp"], {}, {"src/Tests/App.csproj"}, [], "csharp, Tests is a directory name only"), + (["csharp"], {}, {"src/Tests/App.csproj"}, ["csharp"], "csharp, located under Tests/"), + ( + ["csharp"], + {}, + {"test/Specs.csproj"}, + ["csharp"], + "csharp, located as a test, named nothing", + ), + ( + ["csharp"], + {}, + {"src/App/App.csproj", "docs/x.md"}, + [], + "csharp, neither named nor located as a test", + ), + ( + ["python"], + {}, + {"test_app.py", "pyproject.toml"}, + ["python"], + "python, root-level test module", + ), + ( + ["python"], + {}, + {"app_test.py", "pyproject.toml"}, + ["python"], + "python, trailing _test module", + ), + ( + ["python"], + {}, + {"src/app.py", "contest.py"}, + [], + "python, a module merely containing test", + ), ( ["csharp"], {}, From e3d4129c6f81feb14a63e358247d30659914bcf7 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 3 Sep 2026 12:02:25 -0700 Subject: [PATCH 4/4] Require Python in a Test Directory and Drop a Shape-Only Assertion A directory named tests carrying only a README counted as a Python test suite, which is the package-only false claim this change set out to remove. The directory signal now needs a .py file under it, while a module pytest discovers by name still counts wherever it sits. The workflow test asserted that the phrase opening the lockfile branch appears exactly twice. That guards a textual shape rather than a behavior: hoisting the condition into a variable would preserve the branch exactly and fail the line. The assertions kept below it each encode a defect that shipped once in this branch, so their brittleness buys a regression guard that nothing else can give, no local evaluator being able to run a job's bash. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/tests/test_release_guards.py | 1 - spec/audit.py | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_release_guards.py b/scripts/tests/test_release_guards.py index 3cad6aa8..3e43b46f 100755 --- a/scripts/tests/test_release_guards.py +++ b/scripts/tests/test_release_guards.py @@ -252,7 +252,6 @@ def test_validator_python_leg_reaches_a_pip_dependency_repo(self) -> None: self.assertIn('requirement_args+=(-r "$file")', job) self.assertIn('uv pip install "${requirement_args[@]}"', job) self.assertIn(".venv/bin/python -m pytest --cov-report=xml", job) - self.assertEqual(2, job.count("if [ -f uv.lock ]; then")) # One resolve over every requirements file, never one install per file. # The glob sorts the base file last, so a per-file install lets its pins downgrade what the test-requirements file just resolved. diff --git a/spec/audit.py b/spec/audit.py index c41502e1..d94dc58d 100755 --- a/spec/audit.py +++ b/spec/audit.py @@ -238,7 +238,7 @@ def coverage_claiming_types(types, repo_profiles, type_mechanisms, tree): detectors = { "csharp": lambda paths: any(is_csharp_test_project(p) for p in paths), "python": lambda paths: any( - in_test_directory(p) or is_python_test_module(p) for p in paths + (p.endswith(".py") and in_test_directory(p)) or is_python_test_module(p) for p in paths ), } claiming = [] @@ -5247,6 +5247,20 @@ def _selftest(): [], "python, a module merely containing test", ), + ( + ["python"], + {}, + {"tests/README.md", "pyproject.toml"}, + [], + "python, a test directory holding no Python", + ), + ( + ["python"], + {}, + {"tests/conftest.py", "pyproject.toml"}, + ["python"], + "python, a test directory holding Python", + ), ( ["csharp"], {},