Release: baseline + quarantine tooling to make the sharded suites a gate (Phase 1.3/1.4)

script/compliance-baseline.sh

@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+#
+# Turn sharded compliance runs into a regression gate (RELEASE-PLAN Phase 1.3 / 1.4).
+#
+# script/shard-compliance.sh makes the suites runnable; it does not make them a gate. Neither suite
+# is green — VMTest carries ~400 pre-existing failures — so "did it fail?" is always yes and tells
+# you nothing. The gate is the SET DIFFERENCE of failing case NAMES against a committed baseline
+# (AGENTS.md §2).
+#
+#   script/compliance-baseline.sh --collect --suite=VMTest    # shards -> baseline file
+#   script/compliance-baseline.sh --diff    --suite=VMTest    # current shards vs committed baseline
+#
+# Quarantine: test/compliance/quarantine.txt lists cases that flip between runs of the SAME commit.
+# They are excluded from both sides of the diff, because a flaky case in a baseline manufactures
+# phantom regressions — which has already cost real time on the differential corpus (#24226, and an
+# e08_spread "regression" that turned out to be 17/30 vs 17/30 noise).
+#
+# Entry to the quarantine is a BUG, not a resting place: each line carries a reason and an issue.
+set -uo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$REPO_ROOT" || exit 1
+
+SUITE="VMTest"
+MODE=""
+SHARD_DIR="build/compliance-shards"
+BASELINE_DIR="test/compliance/baselines"
+QUARANTINE="test/compliance/quarantine.txt"
+
+for arg in "$@"; do
+    case "$arg" in
+        --suite=*) SUITE="${arg#*=}" ;;
+        --collect) MODE="collect" ;;
+        --diff)    MODE="diff" ;;
+        *) echo "unknown argument: $arg" >&2; exit 2 ;;
+    esac
+done
+
+if [ -z "$MODE" ]; then
+    echo "usage: $0 (--collect|--diff) [--suite=VMTest]" >&2
+    exit 2
+fi
+
+BASELINE="${BASELINE_DIR}/${SUITE}.failing"
+
+# Quarantined names, comments and blanks stripped.
+quarantined() {
+    if [ -f "$QUARANTINE" ]; then
+        sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$QUARANTINE" | grep -v '^$' | LC_ALL=C sort -u
+    fi
+}
+
+# Union of every shard's failing names for this suite.
+current_failing() {
+    local files
+    files=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null | LC_ALL=C sort)
+    if [ -z "$files" ]; then
+        return 1
+    fi
+    # shellcheck disable=SC2086
+    cat $files 2>/dev/null | LC_ALL=C sort -u
+}
+
+# Results from different --shards=N runs must never be mixed: the union would double-count some
+# cases and omit others, and the coverage check would compare against whichever N happened to sort
+# first. Refuse rather than silently produce a wrong baseline.
+shard_coverage() {
+    local widths
+    widths=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null \
+        | sed -E 's/.*-of-([0-9]+)\.failed/\1/' | LC_ALL=C sort -u)
+    local distinct
+    distinct=$(printf '%s\n' "$widths" | grep -c '^[0-9]' || true)
+    if [ "${distinct:-0}" -gt 1 ]; then
+        echo "MIXED $(printf '%s' "$widths" | tr '\n' ',')"
+        return
+    fi
+    local n
+    n=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.failed" 2>/dev/null | wc -l | tr -d ' ')
+    echo "${n:-0} ${widths:-0}"
+}
+
+if ! current_failing > /tmp/.cb_current 2>/dev/null; then
+    echo "compliance-baseline: no ${SUITE} shard results under ${SHARD_DIR}/" >&2
+    echo "  run script/shard-compliance.sh --suite=${SUITE} --shards=N --shard=i for every i first" >&2
+    exit 2
+fi
+
+read -r have want <<< "$(shard_coverage)"
+if [ "$have" = "MIXED" ]; then
+    echo "compliance-baseline: MIXED shard widths present (${want}) under ${SHARD_DIR}/" >&2
+    echo "  Results from different --shards=N runs cannot be unioned. Remove the stale ones." >&2
+    exit 2
+fi
+if [ "$want" -gt 0 ] && [ "$have" -lt "$want" ]; then
+    # A partial set looks exactly like "these cases got fixed" on the next diff. Refuse it.
+    echo "compliance-baseline: INCOMPLETE — ${have} of ${want} shards have results." >&2
+    echo "  Diffing a partial run reports every unrun case as newly-passing. Finish the run first." >&2
+    exit 2
+fi
+
+quarantined > /tmp/.cb_quarantine || true
+LC_ALL=C comm -23 /tmp/.cb_current /tmp/.cb_quarantine > /tmp/.cb_current_net
+
+if [ "$MODE" = "collect" ]; then
+    mkdir -p "$BASELINE_DIR"
+    {
+        echo "# ${SUITE} failing cases — generated by script/compliance-baseline.sh --collect"
+        echo "# Compare by NAME (AGENTS.md §2). Neither suite is green; a count means nothing."
+        echo "# Quarantined cases (test/compliance/quarantine.txt) are excluded."
+        echo "# shards: ${want}   failing: $(wc -l < /tmp/.cb_current_net | tr -d ' ')"
+        cat /tmp/.cb_current_net
+    } > "$BASELINE"
+    echo "compliance-baseline: wrote ${BASELINE} ($(wc -l < /tmp/.cb_current_net | tr -d ' ') failing names, ${want} shards)"
+    exit 0
+fi
+
+if [ ! -f "$BASELINE" ]; then
+    echo "compliance-baseline: no committed baseline at ${BASELINE} — run --collect on a known-good tree first" >&2
+    exit 2
+fi
+
+grep -v '^#' "$BASELINE" | grep -v '^$' | LC_ALL=C sort -u > /tmp/.cb_baseline
+
+regressions=$(LC_ALL=C comm -13 /tmp/.cb_baseline /tmp/.cb_current_net)
+fixes=$(LC_ALL=C comm -23 /tmp/.cb_baseline /tmp/.cb_current_net)
+
+echo "compliance-baseline: ${SUITE}"
+echo "  baseline failing : $(wc -l < /tmp/.cb_baseline | tr -d ' ')"
+echo "  current  failing : $(wc -l < /tmp/.cb_current_net | tr -d ' ')"
+echo "  quarantined      : $(wc -l < /tmp/.cb_quarantine | tr -d ' ')"
+
+if [ -n "$fixes" ]; then
+    echo
+    echo "FIXED (in baseline, passing now) — regenerate the baseline once verified:"
+    printf '  %s\n' $fixes
+fi
+
+if [ -n "$regressions" ]; then
+    echo
+    echo "REGRESSIONS (failing now, not in baseline):" >&2
+    printf '  %s\n' $regressions >&2
+    echo >&2
+    echo "Re-verify each individually before believing it — several compliance cases are" >&2
+    echo "order-dependent, and a case that flips between runs of the same commit belongs in" >&2
+    echo "${QUARANTINE} with an issue, not in the baseline." >&2
+    exit 1
+fi
+
+echo
+echo "no regressions"
+exit 0

test/compliance/quarantine.txt

@@ -0,0 +1,17 @@
+# Compliance cases that flip between runs of the SAME commit.
+#
+# These are excluded from both sides of the baseline diff by script/compliance-baseline.sh, because
+# a flaky case in a baseline manufactures phantom regressions. That has already cost real time: an
+# e08_spread "regression" in the differential corpus turned out to be 17/30 vs 17/30 noise, and two
+# sweeps of the same master disagreed about k06/m04 (#24388).
+#
+# ENTRY HERE IS A BUG, NOT A RESTING PLACE. Every line needs a reason and an issue, and the goal is
+# to empty this file. A case that merely FAILS belongs in the baseline; only a case that both passes
+# and fails on one commit belongs here.
+#
+# Format: <case name>    # reason (#issue)
+#
+# Nothing is listed yet on purpose. RELEASE-PLAN Phase 1.4 names hrtime_*, proc_get_status_basic,
+# interface_abstract_static_call and dnf_return_type_error as suspected flaky, but suspicion is not
+# measurement — each needs two full runs of one commit showing it on one side and not the other
+# before it is quarantined here. Adding them on reputation would hide real regressions.