diff --git a/script/compliance-baseline.sh b/script/compliance-baseline.sh new file mode 100755 index 00000000000..1da4a36e2d2 --- /dev/null +++ b/script/compliance-baseline.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# +# Turn sharded compliance runs into a regression gate (RELEASE-PLAN Phase 1.3 / 1.4). +# +# script/shard-compliance.sh makes the suites runnable; it does not make them a gate. Neither suite +# is green — VMTest carries ~400 pre-existing failures — so "did it fail?" is always yes and tells +# you nothing. The gate is the SET DIFFERENCE of failing case NAMES against a committed baseline +# (AGENTS.md §2). +# +# script/compliance-baseline.sh --collect --suite=VMTest # shards -> baseline file +# script/compliance-baseline.sh --diff --suite=VMTest # current shards vs committed baseline +# +# Quarantine: test/compliance/quarantine.txt lists cases that flip between runs of the SAME commit. +# They are excluded from both sides of the diff, because a flaky case in a baseline manufactures +# phantom regressions — which has already cost real time on the differential corpus (#24226, and an +# e08_spread "regression" that turned out to be 17/30 vs 17/30 noise). +# +# Entry to the quarantine is a BUG, not a resting place: each line carries a reason and an issue. +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" || exit 1 + +SUITE="VMTest" +MODE="" +SHARD_DIR="build/compliance-shards" +BASELINE_DIR="test/compliance/baselines" +QUARANTINE="test/compliance/quarantine.txt" + +for arg in "$@"; do + case "$arg" in + --suite=*) SUITE="${arg#*=}" ;; + --collect) MODE="collect" ;; + --diff) MODE="diff" ;; + *) echo "unknown argument: $arg" >&2; exit 2 ;; + esac +done + +if [ -z "$MODE" ]; then + echo "usage: $0 (--collect|--diff) [--suite=VMTest]" >&2 + exit 2 +fi + +BASELINE="${BASELINE_DIR}/${SUITE}.failing" + +# Quarantined names, comments and blanks stripped. +quarantined() { + if [ -f "$QUARANTINE" ]; then + sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$QUARANTINE" | grep -v '^$' | LC_ALL=C sort -u + fi +} + +# Union of every shard's failing names for this suite. +current_failing() { + local files + files=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null | LC_ALL=C sort) + if [ -z "$files" ]; then + return 1 + fi + # shellcheck disable=SC2086 + cat $files 2>/dev/null | LC_ALL=C sort -u +} + +# Results from different --shards=N runs must never be mixed: the union would double-count some +# cases and omit others, and the coverage check would compare against whichever N happened to sort +# first. Refuse rather than silently produce a wrong baseline. +shard_coverage() { + local widths + widths=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null \ + | sed -E 's/.*-of-([0-9]+)\.failed/\1/' | LC_ALL=C sort -u) + local distinct + distinct=$(printf '%s\n' "$widths" | grep -c '^[0-9]' || true) + if [ "${distinct:-0}" -gt 1 ]; then + echo "MIXED $(printf '%s' "$widths" | tr '\n' ',')" + return + fi + local n + n=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null | wc -l | tr -d ' ') + echo "${n:-0} ${widths:-0}" +} + +if ! current_failing > /tmp/.cb_current 2>/dev/null; then + echo "compliance-baseline: no ${SUITE} shard results under ${SHARD_DIR}/" >&2 + echo " run script/shard-compliance.sh --suite=${SUITE} --shards=N --shard=i for every i first" >&2 + exit 2 +fi + +read -r have want <<< "$(shard_coverage)" +if [ "$have" = "MIXED" ]; then + echo "compliance-baseline: MIXED shard widths present (${want}) under ${SHARD_DIR}/" >&2 + echo " Results from different --shards=N runs cannot be unioned. Remove the stale ones." >&2 + exit 2 +fi +if [ "$want" -gt 0 ] && [ "$have" -lt "$want" ]; then + # A partial set looks exactly like "these cases got fixed" on the next diff. Refuse it. + echo "compliance-baseline: INCOMPLETE — ${have} of ${want} shards have results." >&2 + echo " Diffing a partial run reports every unrun case as newly-passing. Finish the run first." >&2 + exit 2 +fi + +quarantined > /tmp/.cb_quarantine || true +LC_ALL=C comm -23 /tmp/.cb_current /tmp/.cb_quarantine > /tmp/.cb_current_net + +if [ "$MODE" = "collect" ]; then + mkdir -p "$BASELINE_DIR" + { + echo "# ${SUITE} failing cases — generated by script/compliance-baseline.sh --collect" + echo "# Compare by NAME (AGENTS.md §2). Neither suite is green; a count means nothing." + echo "# Quarantined cases (test/compliance/quarantine.txt) are excluded." + echo "# shards: ${want} failing: $(wc -l < /tmp/.cb_current_net | tr -d ' ')" + cat /tmp/.cb_current_net + } > "$BASELINE" + echo "compliance-baseline: wrote ${BASELINE} ($(wc -l < /tmp/.cb_current_net | tr -d ' ') failing names, ${want} shards)" + exit 0 +fi + +if [ ! -f "$BASELINE" ]; then + echo "compliance-baseline: no committed baseline at ${BASELINE} — run --collect on a known-good tree first" >&2 + exit 2 +fi + +grep -v '^#' "$BASELINE" | grep -v '^$' | LC_ALL=C sort -u > /tmp/.cb_baseline + +regressions=$(LC_ALL=C comm -13 /tmp/.cb_baseline /tmp/.cb_current_net) +fixes=$(LC_ALL=C comm -23 /tmp/.cb_baseline /tmp/.cb_current_net) + +echo "compliance-baseline: ${SUITE}" +echo " baseline failing : $(wc -l < /tmp/.cb_baseline | tr -d ' ')" +echo " current failing : $(wc -l < /tmp/.cb_current_net | tr -d ' ')" +echo " quarantined : $(wc -l < /tmp/.cb_quarantine | tr -d ' ')" + +if [ -n "$fixes" ]; then + echo + echo "FIXED (in baseline, passing now) — regenerate the baseline once verified:" + printf ' %s\n' $fixes +fi + +if [ -n "$regressions" ]; then + echo + echo "REGRESSIONS (failing now, not in baseline):" >&2 + printf ' %s\n' $regressions >&2 + echo >&2 + echo "Re-verify each individually before believing it — several compliance cases are" >&2 + echo "order-dependent, and a case that flips between runs of the same commit belongs in" >&2 + echo "${QUARANTINE} with an issue, not in the baseline." >&2 + exit 1 +fi + +echo +echo "no regressions" +exit 0 diff --git a/test/compliance/quarantine.txt b/test/compliance/quarantine.txt new file mode 100644 index 00000000000..dccfe196daa --- /dev/null +++ b/test/compliance/quarantine.txt @@ -0,0 +1,17 @@ +# Compliance cases that flip between runs of the SAME commit. +# +# These are excluded from both sides of the baseline diff by script/compliance-baseline.sh, because +# a flaky case in a baseline manufactures phantom regressions. That has already cost real time: an +# e08_spread "regression" in the differential corpus turned out to be 17/30 vs 17/30 noise, and two +# sweeps of the same master disagreed about k06/m04 (#24388). +# +# ENTRY HERE IS A BUG, NOT A RESTING PLACE. Every line needs a reason and an issue, and the goal is +# to empty this file. A case that merely FAILS belongs in the baseline; only a case that both passes +# and fails on one commit belongs here. +# +# Format: # reason (#issue) +# +# Nothing is listed yet on purpose. RELEASE-PLAN Phase 1.4 names hrtime_*, proc_get_status_basic, +# interface_abstract_static_call and dnf_return_type_error as suspected flaky, but suspicion is not +# measurement — each needs two full runs of one commit showing it on one side and not the other +# before it is quarantined here. Adding them on reputation would hide real regressions.