diff --git a/README.md b/README.md index 9c0da6f3fc4..3263d7653a4 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ | surface | Build identity | Image size ladder | Pinned image budget | thinking on/off | Output quality | fp16 overflow canary | Runner isolation | measured on | |---|---|---|---|---|---|---|---|---| -| **cuda** | green | green | green | green | green | green | green | `0.33.2-dynres-5-g2b95b4a` | +| **cuda** | green | green | green | green | not run | green | green | `0.33.2-dynres-5-g2b95b4a` | | **mlx-cuda** | not run | not run | not run | not run | not run | not run | not run | — | | **mlx-metal** | not run | not run | not run | not run | not run | not run | not run | — | | **apple-silicon-mlx** | not run | not run | not run | not run | not run | not run | not run | — | diff --git a/docs/maxusai/vision-suite/preflight/release_matrix.py b/docs/maxusai/vision-suite/preflight/release_matrix.py index a2910c63988..f9444d9eef3 100644 --- a/docs/maxusai/vision-suite/preflight/release_matrix.py +++ b/docs/maxusai/vision-suite/preflight/release_matrix.py @@ -32,7 +32,17 @@ # matrix that silently stopped seeing this check would read as green. ("Pinned image budget", {"pinned_image_token_budget", "pinned_budget"}), ("thinking on/off", {"think_format"}), - ("Output quality", {"text_baseline", "quality"}), + # The quality verdict the harness actually records is "extraction_quality" + # (checks.check_quality). "quality" is a name nothing has ever emitted, so + # this column never saw the check at all. What it did see was + # "text_baseline", which is not a verdict: preflight.py records it PASS for + # every arch that gets past the prefix probe, and its only other status + # (ERROR) aborts that arch before anything else runs. A check that cannot + # fail can only inflate the group it sits in -- and a group is reported at + # its weakest check -- so it is dropped rather than kept alongside. + # --quality is opt-in, so a run that did not ask for the arm records no + # result here and the cell reads "not run", like any other unmeasured cell. + ("Output quality", {"extraction_quality"}), ("fp16 overflow canary", {"poison_probe"}), ("Runner isolation", {"endpoint_exclusive"}), ] @@ -79,7 +89,8 @@ def main(paths, version=None): runs = [] for p in paths: try: - runs.append(json.load(open(p))) + with open(p) as fh: + runs.append(json.load(fh)) except Exception as exc: # noqa: BLE001 print(f"skipping {p}: {exc}", file=sys.stderr) if not runs: diff --git a/docs/maxusai/vision-suite/preflight/test_verdicts.py b/docs/maxusai/vision-suite/preflight/test_verdicts.py index 5583d0d0b43..1a9af3ab251 100644 --- a/docs/maxusai/vision-suite/preflight/test_verdicts.py +++ b/docs/maxusai/vision-suite/preflight/test_verdicts.py @@ -8,10 +8,14 @@ python3 test_verdicts.py """ +import contextlib +import io import json import os +import re import subprocess import sys +import tempfile import time import tomllib import pathlib @@ -23,6 +27,7 @@ import checks # noqa: E402 import measure_ladder # noqa: E402 import probes # noqa: E402 +import release_matrix # noqa: E402 from checks import FAIL, PASS, SKIP # noqa: E402 SIZES = ["256x144", "512x288", "1024x576", "2048x1152", "3072x1728"] @@ -1234,6 +1239,135 @@ def test_a_declared_probe_must_add_a_degree_of_freedom(self): f"count as the 16:9 ladder, so it adds no coverage") +class TestReleaseMatrixColumns(unittest.TestCase): + """The generated matrix is the fold's headline artifact — it is embedded in + README.md and attached to the release — so a cell that reads green for a + check nobody ran is the exact claim release_matrix.py's docstring exists to + forbid ("a hand-written green badge is worse than none"). + + "Output quality" was mapped to {"text_baseline", "quality"} and read green + on every run ever recorded. Neither name is the quality verdict: + check_quality records "extraction_quality", nothing has ever emitted + "quality", and "text_baseline" is preflight.py's prefix calibration, PASS + for every arch that gets past the probe. A missed recall floor rendered + green; so did a run that never asked for the quality arm. + """ + + COLUMN = "Output quality" + HERE = pathlib.Path(__file__).parent + + # Names no longer emitted by the harness but carried by artifacts recorded + # before a rename. Kept explicit so this test still holds on a lineage + # cherry-picked without runs/, and so adding one is a deliberate act. + RECORDED_ALIASES = {"pinned_budget"} + + def render(self, results, platform="cuda", + version="0.33.2-dynres-5-gsynthet"): + """Render one synthetic artifact and return {column: cell} for its row. + + Goes through main() rather than the group logic directly: the bug was + in the mapping the renderer reads, which a unit test of worst() cannot + see. + """ + run = {"meta": {"platform": platform, "version": version, + "started_utc": "20260904T120000"}, + "results": results} + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "run.json") + with open(path, "w") as fh: + json.dump(run, fh) + out = io.StringIO() + with contextlib.redirect_stdout(out): + release_matrix.main([path]) + columns = [g for g, _ in release_matrix.GROUPS] + ["measured on"] + for line in out.getvalue().splitlines(): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if cells and cells[0] == f"**{platform}**": + self.assertEqual(len(cells) - 1, len(columns), line) + return dict(zip(columns, cells[1:])) + self.fail(f"no row rendered for {platform}:\n{out.getvalue()}") + + @staticmethod + def rec(check, status, arch="gemma4", summary="synthetic"): + return {"check": check, "arch": arch, "status": status, + "summary": summary} + + # A run always carries these; they are what made the false green invisible + # — every other column was legitimately green beside it. + def with_baseline(self, *extra): + return [self.rec("version", PASS, arch=None), + self.rec("text_baseline", PASS), + self.rec("token_ladder", PASS), + self.rec("endpoint_exclusive", PASS, arch=None)] + list(extra) + + def test_a_failed_quality_arm_is_not_green(self): + """The regression. A missed recall floor is a FAIL an operator reads + the matrix to find; it rendered green because the column was watching + two names the check does not use.""" + row = self.render(self.with_baseline( + self.rec("extraction_quality", FAIL, + summary="label_recall 0.42 < 0.7"))) + self.assertEqual(row[self.COLUMN], "**FAIL**") + self.assertEqual(row["Image size ladder"], "green") # unaffected + + def test_no_quality_result_reads_not_run(self): + """--quality is opt-in (preflight.py), so most runs record no quality + verdict at all. Absence is shown, never assumed green — the same rule + the generator applies to a surface with no run.""" + row = self.render(self.with_baseline()) + self.assertEqual(row[self.COLUMN], "not run") + + def test_a_skipped_quality_arm_reads_skipped(self): + """check_quality SKIPs on a lineage carrying preflight/ without + vision_suite.py, and where no thresholds are recorded. Not measured is + not the same as measured and passed.""" + row = self.render(self.with_baseline( + self.rec("extraction_quality", SKIP, + summary="vision_suite.py is not present in this tree"))) + self.assertEqual(row[self.COLUMN], "skipped") + + def test_a_passing_quality_arm_is_green(self): + """The column must still be able to say green — from the check that + actually measured it.""" + row = self.render(self.with_baseline( + self.rec("extraction_quality", PASS, + summary="json_valid=1.00 label_recall=0.83"))) + self.assertEqual(row[self.COLUMN], "green") + + def test_the_prefix_calibration_cannot_carry_a_column(self): + """text_baseline is not a verdict: preflight.py records it PASS for + every arch that reaches the ladder, and its only other status (ERROR) + returns before any further check runs. A check that cannot fail can + only inflate the group it is mapped into.""" + for column, names in release_matrix.GROUPS: + self.assertNotIn("text_baseline", names, column) + + def test_every_mapped_name_is_one_something_records(self): + """The bug in one line: "quality" was a name nothing had ever emitted, + and a column watching only names nobody records reports on nothing. + Rather than re-checking two names by hand, every name in GROUPS must be + one the harness emits today or one a recorded artifact carries.""" + src = "\n".join((self.HERE / f).read_text() + for f in ("checks.py", "preflight.py")) + known = set(re.findall(r'result\(\s*"([a-z0-9_]+)"', src)) + known |= set(re.findall(r'^\s*name = "([a-z0-9_]+)"', src, re.M)) + known |= self.RECORDED_ALIASES + for artifact in sorted((self.HERE / "runs").glob("*.json")): + try: + recorded = json.loads(artifact.read_text()) + except (ValueError, OSError): + continue # a half-written artifact proves nothing here + known |= {r.get("check") for r in recorded.get("results", [])} + self.assertIn("extraction_quality", known, + "checks.py no longer emits the name this test relies on") + for column, names in release_matrix.GROUPS: + for name in sorted(names): + self.assertIn(name, known, + f'the "{column}" column watches for "{name}", ' + f"which neither checks.py/preflight.py emits nor " + f"any recorded run carries") + + # The main block must stay at the END of the file: unittest.main() runs the # classes defined ABOVE it, so a class appended after it silently never runs # as a script — which is exactly what happened to PoisonNodeCorroboration's