diff --git a/.agents/skills/ship-pr/SKILL.md b/.agents/skills/ship-pr/SKILL.md index 4361e1844d..80bd00161c 100644 --- a/.agents/skills/ship-pr/SKILL.md +++ b/.agents/skills/ship-pr/SKILL.md @@ -74,12 +74,10 @@ Coverage and native `windows` / `macos` stay GitHub Actions; ## Push and poll -Required jobs: `qc`, `test`, `windows`, `macos`, `coverage`, and -`mutants (1/4)` through `mutants (4/4)` (`--shard` stays 0–3). `qc` is +Required jobs: `qc`, `test`, `windows`, `macos`, and `coverage`. `qc` is fmt, ast-grep, deny, the script self-tests, clippy, then -nixos-module-eval on one runner. A mutants shard killed at -30 minutes with no `MISSED` warns and passes. `MISSED` before that kill fails the shard. A -finished non-zero `cargo mutants` exit fails. Structural scan is +nixos-module-eval on one runner. Mutants run nightly with workspace +tests (`mutants.yml`); they are not a required check. Structural scan is `./scripts/ast-grep.sh`. `windows` / `macos` are native store plus `--smoke`, not operator zips. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 81a2b46bff..41b5679a8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,13 +26,7 @@ env: # `qc` is one Ubuntu runner: fmt, ast-grep, deny, scripts, clippy, then # nixos-module-eval. test, windows, and macos each have their own runner. # coverage waits for qc and test so a red test does not start it. -# Mutants checks are `mutants (1/4)` through `mutants (4/4)` (`--shard` -# stays 0–3). They wait for the same three. Under 400 changed lines, -# `mutants (1/4)` runs once (`--shard 0/1`); the other three exit 0. -# A 30m kill with no MISSED in the log passes and posts a warning. -# MISSED before that kill fails the shard. A finished non-zero -# cargo-mutants exit fails. -# Weekly sweep stays mutants.yml. +# Mutants are not a PR check. Nightly workspace runs live in mutants.yml. # nixos-module-runtime is a labeled job (nixos-module-runtime.yml), not this file. # Operator binaries are GitHub Releases (release.yml), not this workflow. # @@ -96,6 +90,8 @@ jobs: run: ./scripts/ci-os-smoke.test.sh - name: pr-checks-watch self-test run: ./scripts/pr-checks-watch.test.sh + - name: mutants queue self-test + run: ./scripts/mutants-queue.test.sh - name: overlay-functional runner self-test run: ./scripts/overlay-functional/run.sh.test.sh - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -174,154 +170,6 @@ jobs: - name: store + node smoke run: ./scripts/ci-os-smoke.sh - # PR in-diff. Required checks `mutants (1/4)` through `mutants (4/4)` - # (`--shard` 0–3). Under 400 changed lines, mutants (1/4) runs the - # whole in-diff set (`--shard 0/1`) and the other three exit 0. - # A 30m kill with no MISSED passes and posts a warning. MISSED already - # in that log fails the shard. A finished non-zero exit fails. - # Docs-only diffs exit 0. Weekly sweep stays mutants.yml. Owner: TESTING.md. - mutants: - name: mutants (${{ matrix.n }}/4) - needs: [qc, test] - if: >- - github.event_name == 'pull_request' - && needs.qc.result == 'success' - && needs.test.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 40 - strategy: - fail-fast: false - # Expressions have no + operator; a `+` in ${{ }} invalidates this file. - # n is the check number. shard is the cargo --shard argument. - matrix: - include: - - shard: 0 - n: 1 - - shard: 1 - n: 2 - - shard: 2 - n: 3 - - shard: 3 - n: 4 - env: - # Workflow env disables incremental for rust-cache and denies warnings. - # cargo-mutants rebuilds one mutant at a time; incremental is the point. - # Weekly mutants.yml does not pass -D warnings. A warning-only build - # failure would look caught here and missed on Sunday. - CARGO_INCREMENTAL: "1" - CARGO_TERM_COLOR: never - RUSTFLAGS: "" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - name: rust or workflow diff - id: diff - env: - BASE_REF: ${{ github.base_ref }} - run: | - set -euo pipefail - git diff "origin/${BASE_REF}".. --unified=0 -- \ - crates Cargo.toml Cargo.lock \ - .github/workflows/ci.yml .github/workflows/mutants.yml \ - > git.diff - if [[ ! -s git.diff ]]; then - echo "rust=false" >> "$GITHUB_OUTPUT" - echo "small=false" >> "$GITHUB_OUTPUT" - echo "no rust or workflow diff; shard passes" - exit 0 - fi - echo "rust=true" >> "$GITHUB_OUTPUT" - lines="$(grep -E '^[+-]' git.diff | grep -cvE '^[+-]{3}' || true)" - lines="${lines:-0}" - if [[ "${lines}" -lt 400 ]]; then - echo "small=true" >> "$GITHUB_OUTPUT" - echo "small diff (${lines} changed lines); mutants (1/4) runs all of it (--shard 0/1)" - else - echo "small=false" >> "$GITHUB_OUTPUT" - echo "large diff (${lines} changed lines); four shards" - fi - - uses: dtolnay/rust-toolchain@a5f673d0ba8626c3977bb416a1612774bc82181b # rustc 1.95.0 - if: steps.diff.outputs.rust == 'true' && (steps.diff.outputs.small != 'true' || matrix.shard == 0) - - uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2.87.16 - if: steps.diff.outputs.rust == 'true' && (steps.diff.outputs.small != 'true' || matrix.shard == 0) - with: - tool: cargo-mutants@27.1.0 - - name: in-diff shard - id: run - if: steps.diff.outputs.rust == 'true' && (steps.diff.outputs.small != 'true' || matrix.shard == 0) - run: | - set -euo pipefail - if [[ "${{ steps.diff.outputs.small }}" == "true" ]]; then - shard_arg="0/1" - else - shard_arg="${{ matrix.shard }}/4" - fi - set +e - timeout --signal=TERM --kill-after=60s 30m \ - cargo mutants --workspace --in-diff git.diff \ - --shard "${shard_arg}" --sharding slice -j 2 \ - > mutants-shard.log 2>&1 - ec=$? - set -e - missed="$(grep -c '^MISSED' mutants-shard.log || true)" - echo "missed=${missed}" >> "$GITHUB_OUTPUT" - cat mutants-shard.log - if [[ "$ec" -eq 124 ]]; then - if [[ "$missed" -gt 0 ]]; then - echo "::warning::mutants shard hit 30m with ${missed} MISSED already in the log" - echo "result=timeout-missed" >> "$GITHUB_OUTPUT" - exit 1 - fi - echo "::warning::mutants shard hit the 30m limit with no MISSED in the partial log; counting as success" - echo "result=timeout" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "result=finished" >> "$GITHUB_OUTPUT" - exit "$ec" - - name: summary - if: always() - run: | - { - echo "### mutants ${{ matrix.n }}/4" - echo "Required. Outcome: ${{ steps.run.outcome }}. Result: ${{ steps.run.outputs.result }}. Missed: ${{ steps.run.outputs.missed }}." - echo "A 30m kill with no MISSED warns and passes. MISSED before that kill fails. A finished non-zero exit fails." - } >> "$GITHUB_STEP_SUMMARY" - - name: fail unless success or clean timeout - if: always() && steps.diff.outputs.rust == 'true' - run: | - set -euo pipefail - result="${{ steps.run.outputs.result }}" - outcome="${{ steps.run.outcome }}" - if [[ "${{ steps.diff.outputs.small }}" == "true" && "${{ matrix.shard }}" != "0" ]]; then - echo "small diff; mutants (${{ matrix.n }}/4) does not run cargo mutants" - exit 0 - fi - missed=0 - if [[ -f mutants-shard.log ]]; then - missed="$(grep -c '^MISSED' mutants-shard.log || true)" - fi - if [[ "$result" == "timeout-missed" || ( "$missed" -gt 0 && ( "$result" == "timeout" || "$outcome" == "cancelled" ) ) ]]; then - echo "::warning::mutants shard hit the time limit with ${missed} MISSED already in the log" - exit 1 - fi - if [[ "$result" == "timeout" || "$outcome" == "cancelled" ]]; then - echo "::warning::mutants shard hit the time limit with no MISSED in the partial log; counting as success" - exit 0 - fi - if [[ "$outcome" == "success" && "$result" == "finished" ]]; then - exit 0 - fi - echo "mutants shard failed (outcome=${outcome} result=${result})" - exit 1 - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: mutants-pr-${{ matrix.n }} - path: mutants.out - if-no-files-found: ignore - retention-days: 5 - # Line-coverage 92% floor + CRAP --fail-above 30 (allowlist in .cargo-crap.toml). # Slow; waits for qc and test so a red check does not start # this job. Required. Job+step timeouts so a hung llvm-cov test binary diff --git a/.github/workflows/mutants.yml b/.github/workflows/mutants.yml index c1e35c7ea4..be7ca60414 100644 --- a/.github/workflows/mutants.yml +++ b/.github/workflows/mutants.yml @@ -1,38 +1,74 @@ -# Weekly cargo-mutants. Not a required check. -# PR in-diff shards are ci.yml job `mutants` (required). A 30m kill with -# no MISSED warns and passes. MISSED before that kill fails the shard. -# Must pass --workspace: default-members is rbitcoin-node only. -# Owner: TESTING.md (Mutation testing). Snapshot lists: docs/mutants/. +# Nightly workspace mutants. Not a required check. +# New lines since the cursor run first, then a rotating backlog. +# 90 minute budget, one workspace suite at a time. MISSED is an artifact, +# not a failure. Owner: TESTING.md (Mutation testing). name: mutants on: schedule: - - cron: "0 3 * * 0" + - cron: "0 7 * * *" workflow_dispatch: permissions: contents: read + actions: read jobs: - mutants-weekly: + mutants-nightly: runs-on: ubuntu-latest - timeout-minutes: 360 - strategy: - fail-fast: false - matrix: - shard: [0, 1, 2, 3, 4, 5, 6, 7] + timeout-minutes: 110 + env: + CARGO_INCREMENTAL: "1" + CARGO_TERM_COLOR: never + RUSTFLAGS: "" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 persist-credentials: false - submodules: false - uses: dtolnay/rust-toolchain@a5f673d0ba8626c3977bb416a1612774bc82181b # rustc 1.95.0 - uses: taiki-e/install-action@9114bf4d891761788c546334fd37538eae1bf8b3 # v2.87.16 with: tool: cargo-mutants@27.1.0 - - run: cargo mutants --workspace --shard ${{ matrix.shard }}/8 -j 2 + - name: previous cursor + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + mkdir -p mutants-nightly + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow mutants.yml --limit 20 \ + --json databaseId,status,conclusion \ + --jq '[.[] | select(.status=="completed")][0].databaseId')" + if [[ -n "${run_id}" && "${run_id}" != "null" ]]; then + gh run download "$run_id" --repo "$GITHUB_REPOSITORY" \ + --name mutants-cursor --dir mutants-nightly || true + fi + - name: nightly queue + run: ./scripts/mutants-nightly.sh + - name: summary + if: always() + run: | + { + echo "### mutants nightly" + echo "Not required. Workspace tests. New code first, then the backlog." + if [[ -f mutants-nightly/missed.txt ]]; then + echo + echo '```' + cat mutants-nightly/missed.txt + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: mutants-cursor + path: mutants-nightly/cursor.json + if-no-files-found: error + retention-days: 14 - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: - name: mutants-${{ matrix.shard }} - path: mutants.out + name: mutants-nightly + path: mutants-nightly/ + if-no-files-found: ignore + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index ba719f9761..210fa3cff9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,9 +126,8 @@ Playbooks: [`.agents/skills/core-functional/SKILL.md`](.agents/skills/core-functional/SKILL.md). - Unreleased notes are a new file under `changelog.d/`. Do not edit `CHANGELOG.md` on a feature branch. Owner: [`docs/releases.md`](docs/releases.md). -- Required CI jobs are `qc`, `test`, `windows`, `macos`, `coverage`, and - `mutants (1/4)` through `mutants (4/4)`. `qc` is fmt, ast-grep, deny, - the script self-tests, clippy, then nixos-module-eval. `--shard` is - 0–3. A 30-minute kill with no `MISSED` warns and passes. `MISSED` - before that kill fails the shard. A finished non-zero `cargo mutants` - exit fails. Owner: [`TESTING.md`](TESTING.md). +- Required CI jobs are `qc`, `test`, `windows`, `macos`, and `coverage`. + `qc` is fmt, ast-grep, deny, the script self-tests, clippy, then + nixos-module-eval. Mutants are a nightly workspace oracle + ([`mutants.yml`](.github/workflows/mutants.yml)), not a PR check. + Owner: [`TESTING.md`](TESTING.md). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e399fd72b0..1d0b9d932f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -244,8 +244,7 @@ Do not edit [`CHANGELOG.md`](./CHANGELOG.md) on a feature branch. The release cut folds those files in. Owner: [`docs/releases.md`](./docs/releases.md). Required **jobs** match [`.github/workflows/ci.yml`](./.github/workflows/ci.yml) -(`qc`, `test`, `windows`, `macos`, `coverage`, `mutants (1/4)` … -`mutants (4/4)`). `qc` is fmt, ast-grep, deny, the script self-tests, +(`qc`, `test`, `windows`, `macos`, `coverage`). `qc` is fmt, ast-grep, deny, the script self-tests, clippy, then nixos-module-eval, in that order on one runner. Label **`nixos-module-runtime`** to run the NixOS module qemu test (not eval). That job is not required. diff --git a/TESTING.md b/TESTING.md index 9a245964d1..430a8876d8 100644 --- a/TESTING.md +++ b/TESTING.md @@ -254,7 +254,7 @@ matches scalar in default tests). Owner: [`docs/quality.md`](./docs/quality.md). | **cargo-crap** | After LCOV, `./scripts/coverage.sh` calls `./scripts/coverage-crap.sh` (skip if `cargo-crap` missing). `--fail-above --threshold 30`; `.cargo-crap.toml` allowlists today's production CRAP>30 functions (remove a name when it scores ≤30). Dry-run: `CRAP_DRY_RUN=1 ./scripts/coverage-crap.sh`. Self-test: `./scripts/coverage-crap.test.sh` | Rides required `coverage`. No `--fail-regression` (llvm-cov coverage % jitters per function) | | **coverage ignore / badge** | `./scripts/coverage.test.sh` (filename ignore, Tier A IBD not skipped, 92% floor, Shields JSON). Publish dry-run: `BADGE_DRY_RUN=1 ./scripts/publish-coverage-badge.sh` | `test` job self-test; `coverage` job writes `coverage/badge.json` and, on green `master`, pushes `badges/coverage.json` | | **Miri** | `./scripts/miri.sh` → `cargo +nightly miri test -p rbitcoin-primitives`. Dry-run: `MIRI_DRY_RUN=1 ./scripts/miri.sh`. Self-test: `./scripts/miri.test.sh` | Nightly `miri.yml` (not required). Never `--workspace` | -| **cargo-mutants** | Must pass `--workspace` (`Cargo.toml` `default-members` is `rbitcoin-node` only; without `--workspace` the list is ~7 mutants). PR: `git diff origin/.. --unified=0 -- crates Cargo.toml Cargo.lock .github/workflows/ci.yml .github/workflows/mutants.yml > git.diff` then `cargo mutants --workspace --in-diff git.diff --shard k/4 --sharding slice -j 2` (k is 0–3). Under 400 changed lines the PR job uses `--shard 0/1` once, on check `mutants (1/4)`. Weekly: `cargo mutants --workspace --shard N/8 -j 2`. Snapshot of missed/timeouts: [`docs/mutants/`](docs/mutants/). | PR: `ci.yml` `mutants (1/4)`–`mutants (4/4)` (`--shard` 0–3), required, needs `qc` and `test`. A finished non-zero exit fails. A 30-minute kill with no `MISSED` in the log warns and passes. `MISSED` already in that log fails the shard. No rust or workflow diff: every shard exits 0. Weekly `mutants.yml` Sunday 03:00 UTC, 8 shards, 360min cap (not required). `workflow_dispatch`. | +| **cargo-mutants** | Nightly, not a PR check. `./scripts/mutants-nightly.sh` lists `--workspace` mutants, runs new diff lines first, then walks a cursor through the rest. Each mutant uses `--test-workspace=true` so a higher journey counts, `--baseline=skip`, `-j 1`, 20 minute mutant timeout, 90 minute budget. Cursor artifact `mutants-cursor`. `MISSED` is uploaded and does not fail the run. Snapshot lists in [`docs/mutants/`](docs/mutants/) are host history, not the gate. | `mutants.yml` daily `0 7 * * *` and `workflow_dispatch`. Job timeout 110 minutes. | Artifact silos above are unchanged: ast-grep / Miri dry-run / crap dry-run do not write `target/`. `mutants.out/` is gitignored. @@ -530,6 +530,14 @@ python3 scripts/core-functional/check_inventory.py ## What a mutant kill looks like -The PR gate is the four `mutants` shards in [`ci.yml`](.github/workflows/ci.yml), described in the table above. Do not run a different `cargo mutants` command and treat that as the gate. - -A new production behavior needs a test that fails when that behavior is removed or inverted. Assert the observable result (the error variant, the bytes, the height), not the test name. Concurrency, ordering, and cancellation are not what `cargo-mutants` models; those need a direct test. A diff that does not touch `crates/`, the Cargo manifests, or the mutants workflows does not run mutants. +Mutants are a nightly oracle, not a pull-request check. The command is +`./scripts/mutants-nightly.sh` ([`mutants.yml`](.github/workflows/mutants.yml)). +Do not treat a package-only `cargo mutants` run as the gate: a journey +outside the mutated crate must be able to catch the mutant. + +A new production behavior still needs a test that fails when that behavior +is removed or inverted. Put that assert on a catalog journey, or on a +store-less unit when the journey cannot see it. `MISSED` in the nightly +artifact means no workspace test cared. Extend the journey. A same-crate +twin that only existed to satisfy a package-local mutant is a deletion +candidate once a workspace run shows the journey catching it. diff --git a/changelog.d/mutants-nightly.md b/changelog.d/mutants-nightly.md new file mode 100644 index 0000000000..26cbacbede --- /dev/null +++ b/changelog.d/mutants-nightly.md @@ -0,0 +1,5 @@ +Changed + +- Mutants are a nightly workspace run, not a pull-request check. New + code is first in the queue, then a cursor walks older mutants. A + `MISSED` line is an artifact. It does not fail the night or the PR. diff --git a/docs/releases.md b/docs/releases.md index 2783092a2d..3127647ec2 100644 --- a/docs/releases.md +++ b/docs/releases.md @@ -138,7 +138,8 @@ show as `core-functional / core-functional` and so on. | Check | Who | |-------|-----| -| `qc` `test` `windows` `macos` `coverage` `mutants (1/4)`–`(4/4)` | Every PR (`ci.yml`). `qc` runs fmt, ast-grep, deny, script self-tests, clippy, then nixos-module-eval. A 30-minute mutants kill with no `MISSED` warns and passes. `MISSED` before that kill fails. A finished non-zero exit fails. | +| `qc` `test` `windows` `macos` `coverage` | Every PR (`ci.yml`). `qc` runs fmt, ast-grep, deny, script self-tests, clippy, then nixos-module-eval. | +| `mutants` | Nightly `0 7 * * *` and `workflow_dispatch` (`mutants.yml`). Workspace tests, new code first, then a backlog cursor. 90 minute budget. Not required. `MISSED` is an artifact. | | `core-functional` | Nightly, `workflow_dispatch`, label **`core-functional`**; via `release-gate.yml` on label **`release`** **or** ship version | | `overlay-functional` | Nightly (`42 6`), `workflow_dispatch`, label **`overlay-functional`**; via `release-gate.yml` on label **`release`** **or** ship version | | `warnet-example` | Label **`warnet`**, `workflow_dispatch`; via `release-gate.yml` on label **`release`** **or** ship version. Two-tank Docker lab. | diff --git a/scripts/mutants-nightly.sh b/scripts/mutants-nightly.sh new file mode 100755 index 0000000000..6d1250d171 --- /dev/null +++ b/scripts/mutants-nightly.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# Time-boxed workspace mutants. New lines since the cursor first, then a +# rotating backlog. MISSED is written for humans; it does not fail the run. +# Owner: TESTING.md (Mutation testing). +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" + +# 90 minutes. One workspace suite is the unit of work, and two of them in +# parallel do not fit a hosted runner, so this is -j 1. An hour finishes +# only a couple of mutants after setup; 90 minutes can clear a normal day's +# new mutants and still walk the backlog. +BUDGET_SEC="${MUTANTS_BUDGET_SEC:-5400}" +BATCH="${MUTANTS_BATCH:-4}" +# A mutant that has not finished in 20 minutes is a hang, not a miss. +MUTANT_TIMEOUT="${MUTANTS_TIMEOUT:-1200}" +CURSOR="${MUTANTS_CURSOR:-mutants-nightly/cursor.json}" +OUT="${MUTANTS_OUT:-mutants-nightly}" + +mkdir -p "$OUT" +if [[ ! -f "$CURSOR" ]]; then + printf '%s\n' '{"new_base":"","new_skip":0,"old_index":0}' >"$CURSOR" +fi + +new_base="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("new_base",""))' "$CURSOR")" +new_skip="$(python3 -c 'import json,sys; print(int(json.load(open(sys.argv[1])).get("new_skip",0)))' "$CURSOR")" +old_index="$(python3 -c 'import json,sys; print(int(json.load(open(sys.argv[1])).get("old_index",0)))' "$CURSOR")" + +if [[ -z "$new_base" ]] || ! git rev-parse --verify "${new_base}^{commit}" >/dev/null 2>&1; then + new_base="$(git rev-list -n 1 --before="24 hours ago" HEAD || true)" + if [[ -z "$new_base" ]]; then + new_base="$(git rev-parse HEAD)" + fi + new_skip=0 + python3 - "$CURSOR" "$new_base" <<'PY' +import json, sys +path, base = sys.argv[1], sys.argv[2] +cur = json.load(open(path)) +cur["new_base"] = base +cur["new_skip"] = 0 +json.dump(cur, open(path, "w"), indent=2) +open(path, "a").write("\n") +PY +fi + +head_sha="$(git rev-parse HEAD)" +git diff "$new_base"..HEAD --unified=0 -- crates >"$OUT/new.diff" || true + +echo "mutants-nightly: listing workspace mutants" +cargo mutants --workspace --list >"$OUT/all.txt" + +python3 "$ROOT/scripts/mutants_queue.py" order \ + --list "$OUT/all.txt" --diff "$OUT/new.diff" \ + --old-index "$old_index" --new-skip "$new_skip" >"$OUT/queue.txt" + +read -r n_new n_old < <(python3 - "$OUT/all.txt" "$OUT/new.diff" <<'PY' +import sys +sys.path.insert(0, "scripts") +from mutants_queue import changed_lines, load_mutants, parse_mutant +mutants = load_mutants(open(sys.argv[1]).read()) +changed = changed_lines(open(sys.argv[2]).read()) +n_new = sum(1 for line in mutants if (p := parse_mutant(line)) and (p[0], p[1]) in changed) +print(n_new, len(mutants) - n_new) +PY +) + +echo "mutants-nightly: new=$n_new skip=$new_skip old=$n_old index=$old_index budget=${BUDGET_SEC}s" + +deadline=$((SECONDS + BUDGET_SEC)) +: >"$OUT/missed.txt" +: >"$OUT/ran.txt" +completed_total=0 + +re_args_for_batch() { + local start="$1" count="$2" i line + RE_ARGS=() + i=0 + while IFS= read -r line; do + if ((i >= start && i < start + count)); then + local escaped + escaped="$(python3 -c 'import re,sys; print(re.escape(sys.argv[1]))' "$line")" + RE_ARGS+=(--re "$escaped") + fi + i=$((i + 1)) + done <"$OUT/queue.txt" +} + +queue_len="$(grep -c . "$OUT/queue.txt" || true)" +offset=0 +while ((offset < queue_len && SECONDS < deadline)); do + remain=$((deadline - SECONDS)) + if ((remain < 60)); then + break + fi + re_args_for_batch "$offset" "$BATCH" + if ((${#RE_ARGS[@]} == 0)); then + break + fi + echo "mutants-nightly: batch at $offset (${#RE_ARGS[@]} regexes, ${remain}s left)" + set +e + timeout --signal=TERM --kill-after=60s "$remain" \ + cargo mutants --workspace --test-workspace=true --baseline=skip \ + -j 1 --timeout "$MUTANT_TIMEOUT" \ + "${RE_ARGS[@]}" \ + >"$OUT/batch-$offset.log" 2>&1 + ec=$? + set -e + # Count finished scenarios. A killed batch still advances through those. + done_n="$(grep -cE '^(caught|MISSED|TIMEOUT|unviable)' "$OUT/batch-$offset.log" || true)" + grep -E '^MISSED' "$OUT/batch-$offset.log" >>"$OUT/missed.txt" || true + if ((done_n == 0)); then + echo "mutants-nightly: batch made no progress (exit $ec); same queue next night" + break + fi + python3 "$ROOT/scripts/mutants_queue.py" advance \ + --cursor "$CURSOR" --n-new "$n_new" --n-old "$n_old" \ + --completed "$done_n" --head "$head_sha" + completed_total=$((completed_total + done_n)) + offset=$((offset + done_n)) + if ((ec == 124)); then + echo "mutants-nightly: budget exhausted after $completed_total mutants" + break + fi +done + +cp "$CURSOR" "$OUT/cursor.json" +echo "mutants-nightly: completed=$completed_total missed=$(grep -c . "$OUT/missed.txt" || true)" +if [[ -s "$OUT/missed.txt" ]]; then + echo "mutants-nightly: MISSED (not a failure; extend a journey or keep a guts unit)" + cat "$OUT/missed.txt" +fi diff --git a/scripts/mutants-queue.test.sh b/scripts/mutants-queue.test.sh new file mode 100755 index 0000000000..caafeb4b85 --- /dev/null +++ b/scripts/mutants-queue.test.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Cursor math for the nightly mutants queue. Does not run cargo-mutants. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +PY="$ROOT/scripts/mutants_queue.py" +PASS=0 +FAIL=0 + +assert_ok() { + local name="$1" + shift + if "$@"; then + echo "ok - $name" + PASS=$((PASS + 1)) + else + echo "not ok - $name" + FAIL=$((FAIL + 1)) + fi +} + +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +cat >"$tmp/list.txt" <<'EOF' +crates/a/src/lib.rs:2:1: replace a in old +crates/b/src/lib.rs:4:1: replace b in new +crates/a/src/lib.rs:9:1: replace c in old +not a mutant +EOF + +cat >"$tmp/diff.txt" <<'EOF' +diff --git a/crates/b/src/lib.rs b/crates/b/src/lib.rs +--- a/crates/b/src/lib.rs ++++ b/crates/b/src/lib.rs +@@ -3,0 +4 @@ ++fn new_line() {} +EOF + +order="$(python3 "$PY" order --list "$tmp/list.txt" --diff "$tmp/diff.txt" --old-index 1 --new-skip 0)" +first="$(printf '%s\n' "$order" | head -n 1)" +second="$(printf '%s\n' "$order" | sed -n '2p')" +assert_ok "new mutant is first" test "$first" = "crates/b/src/lib.rs:4:1: replace b in new" +assert_ok "old rotation starts at index 1" test "$second" = "crates/a/src/lib.rs:9:1: replace c in old" + +skipped="$(python3 "$PY" order --list "$tmp/list.txt" --diff "$tmp/diff.txt" --old-index 0 --new-skip 1 | head -n 1)" +assert_ok "new-skip drops the new prefix" test "$skipped" = "crates/a/src/lib.rs:2:1: replace a in old" + +printf '%s\n' '{"new_base":"abc","new_skip":0,"old_index":1}' >"$tmp/cursor.json" +python3 "$PY" advance --cursor "$tmp/cursor.json" --n-new 2 --n-old 2 --completed 1 --head abc +assert_ok "partial new keeps the base" grep -q '"new_base": "abc"' "$tmp/cursor.json" +assert_ok "partial new records skip" grep -q '"new_skip": 1' "$tmp/cursor.json" + +python3 "$PY" advance --cursor "$tmp/cursor.json" --n-new 2 --n-old 2 --completed 2 --head def +assert_ok "finished new moves the base" grep -q '"new_base": "def"' "$tmp/cursor.json" +assert_ok "finished new clears skip" grep -q '"new_skip": 0' "$tmp/cursor.json" +# old_index was 1; the extra completed mutant walks one old slot -> 0 (mod 2) +assert_ok "old index wraps" grep -q '"old_index": 0' "$tmp/cursor.json" + +echo "$PASS passed, $FAIL failed" +test "$FAIL" -eq 0 diff --git a/scripts/mutants_queue.py b/scripts/mutants_queue.py new file mode 100755 index 0000000000..9444373bba --- /dev/null +++ b/scripts/mutants_queue.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Order cargo-mutants names: changed lines first, then a rotating backlog. + +The nightly job runs a time budget of these names with ``--test-workspace``. +A cursor remembers how far the new-code prefix and the old rotation got. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path + +HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") + + +def parse_mutant(line: str) -> tuple[str, int, str] | None: + parts = line.split(":", 3) + if len(parts) < 4: + return None + path, line_no, _col, _rest = parts + try: + return path, int(line_no), line + except ValueError: + return None + + +def changed_lines(diff_text: str) -> set[tuple[str, int]]: + """New-file line numbers touched by a ``git diff -U0``.""" + path = "" + new_line = 0 + touched: set[tuple[str, int]] = set() + for raw in diff_text.splitlines(): + if raw.startswith("+++ b/"): + path = raw[6:] + continue + if raw.startswith("+++ /dev/null"): + path = "" + continue + match = HUNK.match(raw) + if match: + new_line = int(match.group(1)) + continue + if not path or raw.startswith(("diff ", "index ", "--- ")): + continue + if raw.startswith("+"): + touched.add((path, new_line)) + new_line += 1 + elif raw.startswith("-"): + continue + elif raw.startswith("\\"): + continue + else: + new_line += 1 + return touched + + +def load_mutants(text: str) -> list[str]: + out = [] + for raw in text.splitlines(): + line = raw.strip() + if parse_mutant(line): + out.append(line) + return out + + +def order(mutants: list[str], changed: set[tuple[str, int]], old_index: int) -> tuple[list[str], list[str]]: + new: list[str] = [] + old: list[str] = [] + for line in mutants: + parsed = parse_mutant(line) + if parsed and (parsed[0], parsed[1]) in changed: + new.append(line) + else: + old.append(line) + if old: + start = old_index % len(old) + old = old[start:] + old[:start] + return new, old + + +def advance(new_skip: int, old_index: int, n_new: int, n_old: int, completed: int) -> tuple[int, int, bool]: + """Return (new_skip, old_index, new_done). + + ``new_done`` means every new mutant was attempted, so the caller can + move ``new_base`` to HEAD. + """ + if completed < 0: + raise ValueError("completed") + take_new = min(completed, max(n_new - new_skip, 0)) + new_skip += take_new + rest = completed - take_new + if n_old and rest: + old_index = (old_index + rest) % n_old + elif n_old == 0: + old_index = 0 + return new_skip, old_index, new_skip >= n_new + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="cmd", required=True) + + order_p = sub.add_parser("order") + order_p.add_argument("--list", type=Path, required=True) + order_p.add_argument("--diff", type=Path, required=True) + order_p.add_argument("--old-index", type=int, required=True) + order_p.add_argument("--new-skip", type=int, default=0) + + adv = sub.add_parser("advance") + adv.add_argument("--cursor", type=Path, required=True) + adv.add_argument("--n-new", type=int, required=True) + adv.add_argument("--n-old", type=int, required=True) + adv.add_argument("--completed", type=int, required=True) + adv.add_argument("--head", required=True) + + args = parser.parse_args(argv) + if args.cmd == "order": + mutants = load_mutants(args.list.read_text()) + changed = changed_lines(args.diff.read_text()) if args.diff.stat().st_size else set() + new, old = order(mutants, changed, args.old_index) + if args.new_skip: + new = new[args.new_skip :] + for line in new + old: + print(line) + return 0 + + cursor = json.loads(args.cursor.read_text()) + new_skip, old_index, done = advance( + int(cursor.get("new_skip", 0)), + int(cursor.get("old_index", 0)), + args.n_new, + args.n_old, + args.completed, + ) + if done: + cursor["new_base"] = args.head + cursor["new_skip"] = 0 + else: + cursor["new_skip"] = new_skip + cursor["old_index"] = old_index + args.cursor.write_text(json.dumps(cursor, indent=2) + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/pr-checks-watch.sh b/scripts/pr-checks-watch.sh index 8e1a71d893..2d580cbfbf 100755 --- a/scripts/pr-checks-watch.sh +++ b/scripts/pr-checks-watch.sh @@ -17,10 +17,6 @@ REQUIRED=( windows macos coverage - "mutants (1/4)" - "mutants (2/4)" - "mutants (3/4)" - "mutants (4/4)" ) usage() { diff --git a/scripts/pr-checks-watch.test.sh b/scripts/pr-checks-watch.test.sh index 8040f33ed1..588a525a4c 100755 --- a/scripts/pr-checks-watch.test.sh +++ b/scripts/pr-checks-watch.test.sh @@ -41,11 +41,7 @@ all_green="$( test pass 2m \ windows pass 1m \ macos pass 50s \ - coverage pass 2m \ - 'mutants (1/4)' pass 2m \ - 'mutants (2/4)' pass 2m \ - 'mutants (3/4)' pass 2m \ - 'mutants (4/4)' pass 2m + coverage pass 2m )" rc=0 out="$(CI_PR_CHECKS_TEXT="$all_green" "$RUN" --once 2>&1)" || rc=$? @@ -59,10 +55,6 @@ analyze_pending_windows_green="$( windows pass 1m \ macos pass 50s \ coverage pass 2m \ - 'mutants (1/4)' pass 2m \ - 'mutants (2/4)' pass 2m \ - 'mutants (3/4)' pass 2m \ - 'mutants (4/4)' pass 2m \ 'Analyze (rust)' pending 0 )" rc=0