CI: select the compliance suite; add a spine-chunk split-TU probe

.github/workflows/compliance-nightly.yml

@@ -11,6 +11,10 @@
 #
 # Why the diff and not the exit status: neither suite is green (~422 known failures), so "did it
 # fail?" is always yes and carries no information. Only new NAMES matter (AGENTS.md §2).
+#
+# The suite is selectable on workflow_dispatch. The cron nightly runs VMTest, which is the suite
+# with a committed baseline; JITTest (~472 failures) has none yet, so `--suite=JITTest --diff` is
+# expected to refuse until a collect run has been reviewed and landed by a human.
 name: Compliance nightly
 
 on:
@@ -25,14 +29,30 @@ on:
         default: 'diff'
         type: choice
         options: ['diff', 'collect']
+      suite:
+        description: 'which compliance suite to run'
+        required: false
+        default: 'VMTest'
+        type: choice
+        options: ['VMTest', 'JITTest']
 
 concurrency:
-  group: compliance-nightly-${{ github.ref }}
-  cancel-in-progress: true
+  # Keyed on the suite too: a JITTest collect must not cancel a VMTest nightly on the same ref.
+  # On `schedule` there are no inputs, so this falls back to VMTest — which is what the cron runs.
+  group: compliance-nightly-${{ github.ref }}-${{ github.event.inputs.suite || 'VMTest' }}
+  # Never cancel a manual run. The two-run baseline protocol needs two collects of the SAME commit,
+  # and with cancel-in-progress those two runs kill each other the moment they overlap — leaving one
+  # result and no way to tell flaky from changed. Scheduled runs still supersede each other.
+  cancel-in-progress: ${{ github.event_name == 'schedule' }}
+
+env:
+  # Single source of truth for the suite. `schedule` supplies no inputs, so the cron nightly is
+  # VMTest; workflow_dispatch can select JITTest to collect its (uncommitted, ~472-failure) baseline.
+  SUITE: ${{ github.event.inputs.suite || 'VMTest' }}
 
 jobs:
   shard:
-    name: VMTest shard ${{ matrix.shard }}
+    name: ${{ github.event.inputs.suite || 'VMTest' }} shard ${{ matrix.shard }}
     runs-on: ubuntu-22.04
     timeout-minutes: 60
     strategy:
@@ -55,12 +75,13 @@ jobs:
             -e PHP_COMPILER_MEMORY_LIMIT=1536M \
             -e PHP_COMPILER_LLVM_MEMORY_LIMIT=4096M \
             -e SHARD_TIMEOUT=900 \
+            -e SUITE="$SUITE" \
             -v "$PWD:/compiler" -w /compiler \
             php-compiler:22.04-dev bash -lc '
               set -uo pipefail
               composer install --no-interaction --ignore-platform-reqs -q
               script/apply-patches.sh
-              script/shard-compliance.sh --suite=VMTest --shards=24 --shard=${{ matrix.shard }}
+              script/shard-compliance.sh --suite="$SUITE" --shards=24 --shard=${{ matrix.shard }}
             '
 
       # Upload even on failure: a shard that times out still produces partial results plus a
@@ -71,8 +92,9 @@ jobs:
         with:
           name: shard-${{ matrix.shard }}
           path: |
-            build/compliance-shards/VMTest-${{ matrix.shard }}-of-24.failed
-            build/compliance-shards/VMTest-${{ matrix.shard }}-of-24.timeout
+            build/compliance-shards/${{ github.event.inputs.suite || 'VMTest' }}-${{ matrix.shard }}-of-24.failed
+            build/compliance-shards/${{ github.event.inputs.suite || 'VMTest' }}-${{ matrix.shard }}-of-24.executed
+            build/compliance-shards/${{ github.event.inputs.suite || 'VMTest' }}-${{ matrix.shard }}-of-24.timeout
           if-no-files-found: warn
           retention-days: 14
 
@@ -99,7 +121,7 @@ jobs:
           # compliance-baseline.sh refuses a partial set, a mixed --shards=N set, or any set
           # containing a .timeout marker — so reaching a verdict at all means the run was whole.
           # It exits non-zero only on NEW failing names, never on the ~422 known ones.
-          bash script/compliance-baseline.sh --diff --suite=VMTest
+          bash script/compliance-baseline.sh --diff --suite="$SUITE"
 
       # A baseline is only valid for the environment that produced it (#24726). The first nightly
       # diffed a dev-box baseline from a runner and reported 23 "regressions" that conflate genuine
@@ -108,18 +130,35 @@ jobs:
       #
       # Run it TWICE and quarantine anything that flips: a single run was measured wrong in 9 places
       # (#24672). The artifact is uploaded rather than committed — a human decides what lands.
+      #
+      # BOTH RUNS MUST BE THE SAME COMMIT. The quarantine landed in #24778 was derived from two
+      # collects 15 commits apart, so every flip it recorded was a CHANGE, not flakiness: 0ae0f941c
+      # (highlight_string wire format) fixed four highlight_* cases and broke three, and 6af97e0b4
+      # (property hooks on the default profile) made six property_hook_* cases start executing at
+      # all — VMTest.php gates them on CompilerVersion::supportsPropertyHooks(), so before that
+      # commit the provider never yielded them. A case that goes from unyielded to failing is
+      # indistinguishable from a flaky one in a diff of failing NAMES. Result: 18 real failures were
+      # quarantined into invisibility (#24726).
+      #
+      # workflow_dispatch resolves a branch to its head AT DISPATCH TIME, so two dispatches minutes
+      # apart on a branch that merges ~4 commits/hour are not the same tree. Cut a ref that does not
+      # move and dispatch both runs against it.
       - name: Collect a baseline from this environment
         if: github.event.inputs.mode == 'collect'
         run: |
           ls -la build/compliance-shards/ || true
-          bash script/compliance-baseline.sh --collect --suite=VMTest
+          bash script/compliance-baseline.sh --collect --suite="$SUITE"
           echo "--- collected ---"
-          head -5 test/compliance/baselines/VMTest.failing
+          head -5 "test/compliance/baselines/${SUITE}.failing"
 
       - name: Upload collected baseline
         if: github.event.inputs.mode == 'collect'
         uses: actions/upload-artifact@v4
         with:
-          name: VMTest-baseline-${{ github.sha }}
-          path: test/compliance/baselines/VMTest.failing
+          name: ${{ github.event.inputs.suite || 'VMTest' }}-baseline-${{ github.sha }}
+          # Both files. Without the .executed companion the next diff cannot tell a fixed case from
+          # one the provider stopped yielding, which is how 18 real failures got buried (#24726).
+          path: |
+            test/compliance/baselines/${{ github.event.inputs.suite || 'VMTest' }}.failing
+            test/compliance/baselines/${{ github.event.inputs.suite || 'VMTest' }}.executed
           retention-days: 30

script/compliance-baseline.sh

@@ -23,9 +23,10 @@ cd "$REPO_ROOT" || exit 1
 
 SUITE="VMTest"
 MODE=""
-SHARD_DIR="build/compliance-shards"
-BASELINE_DIR="test/compliance/baselines"
-QUARANTINE="test/compliance/quarantine.txt"
+# Overridable for tests (#24498 does the same for the case corpus); production uses the defaults.
+: "${SHARD_DIR:=build/compliance-shards}"
+: "${BASELINE_DIR:=test/compliance/baselines}"
+: "${QUARANTINE:=test/compliance/quarantine.txt}"
 
 for arg in "$@"; do
     case "$arg" in
@@ -42,6 +43,7 @@ if [ -z "$MODE" ]; then
 fi
 
 BASELINE="${BASELINE_DIR}/${SUITE}.failing"
+EXECUTED_BASELINE="${BASELINE_DIR}/${SUITE}.executed"
 
 # Quarantined names, comments and blanks stripped.
 quarantined() {
@@ -69,6 +71,28 @@ current_failing() {
         | LC_ALL=C sort -u
 }
 
+# Union of every shard's EXECUTED names, same normalisation.
+#
+# Failing names alone cannot tell "this case was fixed" from "this case is no longer run". VMTest
+# gates cases on CompilerVersion::supports*(), and the default profile has flipped repeatedly
+# (6af97e0b4 enabled property hooks, 910820500 re-rejected them; 270fea4ef re-gated asymmetric
+# visibility). Each flip changes the yielded set, and a case that vanishes reads as FIXED — so
+# regenerating the baseline drops it from tracking silently. Measured at a634624db: four cases the
+# diff called FIXED (asymmetric_visibility_write, asymmetric_visibility_public_protected_set,
+# static_asymmetric_visibility, match_typed_class_const) do not execute at all.
+current_executed() {
+    local files
+    files=$(find "$SHARD_DIR" -name "${SUITE}-*-of-*.executed" 2>/dev/null | LC_ALL=C sort)
+    if [ -z "$files" ]; then
+        return 1
+    fi
+    # shellcheck disable=SC2086
+    cat $files 2>/dev/null \
+        | sed -e 's/^.*data set "//' -e 's/"$//' \
+        | grep -v '^$' \
+        | LC_ALL=C sort -u
+}
+
 # Results from different --shards=N runs must never be mixed: the union would double-count some
 # cases and omit others, and the coverage check would compare against whichever N happened to sort
 # first. Refuse rather than silently produce a wrong baseline.
@@ -121,6 +145,8 @@ fi
 quarantined > /tmp/.cb_quarantine || true
 LC_ALL=C comm -23 /tmp/.cb_current /tmp/.cb_quarantine > /tmp/.cb_current_net
 
+current_executed > /tmp/.cb_executed 2>/dev/null || : > /tmp/.cb_executed
+
 if [ "$MODE" = "collect" ]; then
     mkdir -p "$BASELINE_DIR"
     {
@@ -131,6 +157,20 @@ if [ "$MODE" = "collect" ]; then
         cat /tmp/.cb_current_net
     } > "$BASELINE"
     echo "compliance-baseline: wrote ${BASELINE} ($(wc -l < /tmp/.cb_current_net | tr -d ' ') failing names, ${want} shards)"
+
+    if [ -s /tmp/.cb_executed ]; then
+        {
+            echo "# ${SUITE} EXECUTED cases — generated by script/compliance-baseline.sh --collect"
+            echo "# Not a pass list. This records which cases the provider yielded, so that a case"
+            echo "# which stops being yielded is reported as DROPPED rather than silently as FIXED."
+            echo "# shards: ${want}   executed: $(wc -l < /tmp/.cb_executed | tr -d ' ')"
+            cat /tmp/.cb_executed
+        } > "$EXECUTED_BASELINE"
+        echo "compliance-baseline: wrote ${EXECUTED_BASELINE} ($(wc -l < /tmp/.cb_executed | tr -d ' ') executed names)"
+    else
+        echo "compliance-baseline: WARNING — no .executed shard files found; skipping ${EXECUTED_BASELINE}." >&2
+        echo "  Without it the next diff cannot tell a fixed case from one that stopped running." >&2
+    fi
     exit 0
 fi
 
@@ -149,10 +189,60 @@ echo "  baseline failing : $(wc -l < /tmp/.cb_baseline | tr -d ' ')"
 echo "  current  failing : $(wc -l < /tmp/.cb_current_net | tr -d ' ')"
 echo "  quarantined      : $(wc -l < /tmp/.cb_quarantine | tr -d ' ')"
 
-if [ -n "$fixes" ]; then
+# Split the apparent fixes: a case that is no longer executed did not get fixed, it disappeared.
+# Reporting both under "FIXED" is how real failures get dropped from the baseline (#24726).
+#
+# Splitting FIXED from DROPPED needs only THIS run's executed set. The executed BASELINE is a
+# separate question (what is newly executed), so the two are gated independently — conflating them
+# once printed "executed, passing now" over a list that had never been checked for execution.
+dropped=""
+real_fixes="$fixes"
+can_split=0
+if [ -s /tmp/.cb_executed ]; then
+    can_split=1
+    printf '%s\n' $fixes | grep -v '^$' | LC_ALL=C sort -u > /tmp/.cb_fixes
+    dropped=$(LC_ALL=C comm -23 /tmp/.cb_fixes /tmp/.cb_executed)
+    real_fixes=$(LC_ALL=C comm -12 /tmp/.cb_fixes /tmp/.cb_executed)
+fi
+
+if [ -n "$real_fixes" ]; then
+    echo
+    if [ "$can_split" -eq 1 ]; then
+        echo "FIXED (in baseline, executed, passing now) — regenerate the baseline once verified:"
+    else
+        echo "FIXED OR DROPPED (in baseline, not failing now) — execution NOT verified:"
+    fi
+    printf '  %s\n' $real_fixes
+    if [ "$can_split" -eq 0 ]; then
+        echo
+        echo "  No .executed shard files in ${SHARD_DIR}, so this list may contain cases that were"
+        echo "  never run rather than fixed. Do not regenerate the baseline from it."
+    fi
+fi
+
+if [ -n "$dropped" ]; then
+    echo
+    echo "DROPPED (in baseline, NOT EXECUTED in this run) — these were not fixed:"
+    printf '  %s\n' $dropped
+    echo
+    echo "  The provider stopped yielding them. VMTest gates cases on CompilerVersion::supports*(),"
+    echo "  so a profile change silently removes cases from the corpus. Regenerating the baseline"
+    echo "  now would drop them from tracking permanently — decide deliberately, per case."
+fi
+
+if [ "$can_split" -eq 1 ] && [ -f "$EXECUTED_BASELINE" ]; then
+    grep -v '^#' "$EXECUTED_BASELINE" | grep -v '^$' | LC_ALL=C sort -u > /tmp/.cb_exec_baseline
+    newly_run=$(LC_ALL=C comm -13 /tmp/.cb_exec_baseline /tmp/.cb_executed)
+    if [ -n "$newly_run" ]; then
+        echo
+        echo "NEWLY EXECUTED (not in the executed baseline) — $(printf '%s\n' $newly_run | wc -l | tr -d ' ') case(s)."
+        echo "  A profile change enabled them. Any that fail appear as regressions below but are"
+        echo "  new coverage, not new breakage — attribute them before believing the regression."
+    fi
+elif [ "$can_split" -eq 1 ]; then
     echo
-    echo "FIXED (in baseline, passing now) — regenerate the baseline once verified:"
-    printf '  %s\n' $fixes
+    echo "NOTE: no ${EXECUTED_BASELINE} — FIXED/DROPPED is split, but newly-executed cases cannot"
+    echo "  be flagged. Run --collect to establish it."
 fi
 
 if [ -n "$regressions" ]; then

script/shard-compliance.sh

@@ -136,6 +136,7 @@ LOG_DIR="build/compliance-shards"
 mkdir -p "$LOG_DIR"
 LOG="${LOG_DIR}/${SUITE}-${SHARD}-of-${SHARDS}.log"
 FAILED_FILE="${LOG_DIR}/${SUITE}-${SHARD}-of-${SHARDS}.failed"
+EXECUTED_FILE="${LOG_DIR}/${SUITE}-${SHARD}-of-${SHARDS}.executed"
 
 echo "shard-compliance: ${SUITE} shard ${SHARD}/${SHARDS} — ${count} of ${total} cases, timeout ${SHARD_TIMEOUT}s"
 
@@ -154,6 +155,14 @@ rc=$?
 # the comment.
 executed="$(grep -c "^##teamcity\[testStarted" "$LOG" 2>/dev/null || true)"
 executed="${executed:-0}"
+# The executed NAMES, not just the count. VMTest::providePHPTests() gates cases on
+# CompilerVersion::supports*(), so a profile-flipping commit silently changes which cases the
+# provider yields at all. Without this set, a case that stopped executing is indistinguishable from
+# one that started passing, and regenerating the baseline drops it from tracking forever — which is
+# how the #24778 quarantine buried 18 real failures (#24726).
+grep -oE "^##teamcity\[testStarted name='[^']*'" "$LOG" 2>/dev/null \
+    | sed "s/.*name='//; s/'$//" \
+    | LC_ALL=C sort -u > "$EXECUTED_FILE"
 grep -oE "^##teamcity\[testFailed name='[^']*'" "$LOG" 2>/dev/null \
     | sed "s/.*name='//; s/'$//" \
     | LC_ALL=C sort -u > "$FAILED_FILE"