test(rocm): add the gfx1151 correctness matrix and its gate

Makefile

@@ -290,6 +290,12 @@ test-fast-cuda: ## Run tests under the fast dev-iteration profile with CUDA (Lin
 	$(CARGO) test --profile test-fast --features cuda $(FILTER) -- --test-threads=1
 	@echo "$(GREEN)All tests passed!$(RESET)"
 
+.PHONY: test-fast-rocm
+test-fast-rocm: ## Run tests under the fast dev-iteration profile with ROCm (Linux/AMD, experimental; set FILTER=<path>)
+	@echo "$(CYAN)Running tests (test-fast profile, ROCm)...$(RESET)"
+	$(CARGO) test --profile test-fast --features rocm $(FILTER) -- --test-threads=1
+	@echo "$(GREEN)All tests passed!$(RESET)"
+
 .PHONY: check-fast
 check-fast: ## Check all targets under the fast dev-iteration profile (issue #809)
 	@echo "$(CYAN)Checking code (test-fast profile)...$(RESET)"
@@ -715,6 +721,11 @@ verify-test-cuda: ## CUDA gate: cargo test --workspace --profile test-fast --fea
 	@echo "$(CYAN)[verify] test (workspace, test-fast profile, features=cuda, single threaded)...$(RESET)"
 	$(CARGO) test --workspace --profile test-fast --features cuda --no-fail-fast -- --test-threads=1
 
+.PHONY: verify-test-rocm
+verify-test-rocm: ## ROCm gate (experimental): cargo test --workspace --profile test-fast --features rocm --no-fail-fast -- --test-threads=1 (issue #1809)
+	@echo "$(CYAN)[verify] test (workspace, test-fast profile, features=rocm, single threaded)...$(RESET)"
+	$(CARGO) test --workspace --profile test-fast --features rocm --no-fail-fast -- --test-threads=1
+
 .PHONY: verify-versions
 verify-versions: ## Assert every version-tracking workspace crate carries the root `mlxcel` version
 	@echo "$(CYAN)[verify] workspace crate versions...$(RESET)"

benchmarks/logit_traces/metal_m1u_bec64748/METADATA.txt

@@ -0,0 +1,11 @@
+host: Apple M1 Ultra / 128 GiB
+os: macOS 27.0 (26A428)
+mlxcel_commit: bec64748a5a1ac8112cbaebded0252d4d04fbb24
+mlx_pin: 81ba1c6a0e50a9268b931579c2d4f1158b9aab5a
+metallib_sha256: dca1bb428fd549cdc38060af471792a3901f5453c722cf8beacef45cec8e07d8
+logit_trace_sha256: c7d0b533215967645b1bccd0956f41a13027347359bfa7829c51dcce29176edd
+corpus: tests/fixtures/wikitext2_excerpt.txt sha256 16721fc1057d69efc8493718fcad562de9e4554ce05369876cd06dcba431394e
+args: w1='1 128 8 0' w8='8 80 8 512' w256='256 2 8 0' (CHUNK_TOKENS MAX_CHUNKS TOPK PREFILL)
+env: MLXCEL_KEEP_BF16 unset (all four checkpoints are affine 4-bit, bf16 scales kept on both backends)
+tags: 'default' = MLXCEL_FUSED_MOE unset (fused MoE ON for the MoE models); 'fused0' = MLXCEL_FUSED_MOE=0 (gather_qmm path)
+metal path: M1 Ultra is Apple GPU gen 13, so no NAX path is taken

benchmarks/logit_traces/metal_m1u_bec64748/README.md

@@ -0,0 +1,33 @@
+# Metal baseline logit traces for lablup/mlxcel#1809
+
+Teacher-forced logit traces from `examples/logit_trace` on an M1 Ultra (Apple GPU generation 13, so no NAX path), measured at mlxcel `bec64748` with the MLX pin `81ba1c6a`. They are the Metal reference for the cross-backend correctness matrix in lablup/mlxcel#1809; the ROCm traces are produced from the same commit, corpus, and arguments. `METADATA.txt` records the host, OS, pin, and the metallib, binary, and corpus hashes. `RUNS.txt` has the exit status and row count of every run, and `SHA256SUMS` covers every trace.
+
+## What was traced
+
+Four affine 4-bit checkpoints: `qwen3-0.6b-4bit`, `meta-llama-3.1-8b-instruct-4bit`, `qwen3-30b-a3b-4bit`, and `mixtral-8x7b-instruct-v0.1-4bit`. Each is traced over `tests/fixtures/wikitext2_excerpt.txt` at three shapes (`CHUNK_TOKENS MAX_CHUNKS TOPK PREFILL`):
+
+| Tag | Arguments | Positions | Shape it measures |
+|---|---|---|---|
+| `w1` | `1 128 8 0` | 128 | decode |
+| `w8` | `8 80 8 512` | 640 | verify-sized forward behind 512 tokens of context |
+| `w256` | `256 2 8 0` | 512 | prefill |
+
+`default` leaves `MLXCEL_FUSED_MOE` unset, so the two MoE checkpoints run the fused MoE kernel where it applies. `fused0` sets `MLXCEL_FUSED_MOE=0` and takes the `gather_qmm` path. Pick the variant that matches the path the ROCm build takes. Mixtral (top-k 2, 8 experts) sorts its expert indices only when `n_tokens * top_k >= 64` (`src/models/switch_layers.rs`), so its `w256` trace is the one that passes through the sorted `gather_qmm` path.
+
+The `default` traces of the two MoE checkpoints (six files) are not used by the gfx1151 matrix in `docs/benchmark_results/rocm-correctness-gfx1151-2026-09-12.md`, because the fused MoE kernel has no ROCm port and the ROCm side had to run `fused0`. They are kept for two uses: as the same-backend control that isolates a kernel swap from a hardware change, and as the reference to compare against once lablup/mlxcel#1803 gives ROCm a fused MoE kernel.
+
+Absolute local paths in the trace headers were rewritten to repository-relative ones. `scripts/compare_logit_traces.py` reads only the position count and the target token of each row, so the rewrite does not affect a comparison.
+
+## Comparing
+
+```bash
+python3 scripts/compare_logit_traces.py \
+    benchmarks/logit_traces/metal_m1u_bec64748/metal_m1u_bec64748_<model>_<variant>_<tag>.tsv \
+    <rocm trace for the same model, variant and tag>
+```
+
+Gate on disagreement at decided positions (`--decided 2.0`, the default), as lablup/mlxcel#1809 specifies. Byte identity is not expected across backends.
+
+For scale, fused MoE against `gather_qmm` on this same host differs at one position out of 128 for `qwen3-30b-a3b` at `w1`, and that position is undecided. The other five MoE pairs agree at every position. A kernel swap within one backend moves almost nothing here, which is the context for reading a cross-backend disagreement.
+
+`logit_trace` does not prepend BOS to chunk 0 (lablup/mlxcel#1785). Both backends run the same code, so this does not bias a comparison, but it does shift absolute perplexity.

benchmarks/logit_traces/metal_m1u_bec64748/RUNS.txt

@@ -0,0 +1,19 @@
+metal_m1u_bec64748_qwen3-0.6b_default_w1                               exit=0 rows=132 elapsed=3s
+metal_m1u_bec64748_qwen3-0.6b_default_w8                               exit=0 rows=644 elapsed=7s
+metal_m1u_bec64748_qwen3-0.6b_default_w256                             exit=0 rows=516 elapsed=0s
+metal_m1u_bec64748_llama-3.1-8b-instruct_default_w1                    exit=0 rows=132 elapsed=4s
+metal_m1u_bec64748_llama-3.1-8b-instruct_default_w8                    exit=0 rows=644 elapsed=40s
+metal_m1u_bec64748_llama-3.1-8b-instruct_default_w256                  exit=0 rows=516 elapsed=1s
+metal_m1u_bec64748_qwen3-30b-a3b_default_w1                            exit=0 rows=132 elapsed=9s
+metal_m1u_bec64748_qwen3-30b-a3b_default_w8                            exit=0 rows=644 elapsed=38s
+metal_m1u_bec64748_qwen3-30b-a3b_default_w256                          exit=0 rows=516 elapsed=1s
+metal_m1u_bec64748_qwen3-30b-a3b_fused0_w1                             exit=0 rows=132 elapsed=4s
+metal_m1u_bec64748_qwen3-30b-a3b_fused0_w8                             exit=0 rows=644 elapsed=38s
+metal_m1u_bec64748_qwen3-30b-a3b_fused0_w256                           exit=0 rows=516 elapsed=2s
+metal_m1u_bec64748_mixtral-8x7b-instruct_default_w1                    exit=0 rows=132 elapsed=13s
+metal_m1u_bec64748_mixtral-8x7b-instruct_default_w8                    exit=0 rows=644 elapsed=103s
+metal_m1u_bec64748_mixtral-8x7b-instruct_default_w256                  exit=0 rows=516 elapsed=3s
+metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w1                     exit=0 rows=132 elapsed=6s
+metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w8                     exit=0 rows=644 elapsed=104s
+metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w256                   exit=0 rows=516 elapsed=3s
+ALL_DONE

benchmarks/logit_traces/metal_m1u_bec64748/SHA256SUMS

@@ -0,0 +1,18 @@
+d92967af65e59bf228c4ad681cc1971f3b11c4dc051e53929c4ea22ff7776d53  metal_m1u_bec64748_llama-3.1-8b-instruct_default_w1.tsv
+8e4c40f1dc0e1e9992f8ba9416f7a6310b5cd39360fa66607c16a1353c18e976  metal_m1u_bec64748_llama-3.1-8b-instruct_default_w256.tsv
+0401f769b31271ff0af98f3346ced1b0cc286a2c75a23a1c9c3ddc0129bb3015  metal_m1u_bec64748_llama-3.1-8b-instruct_default_w8.tsv
+828ab85e81b83f278cd41d9e0cd589b476e909d6fb781f4453ed73a6103cc18b  metal_m1u_bec64748_mixtral-8x7b-instruct_default_w1.tsv
+6bc8cd8b028c616c2145ecf1b97dc1c8dae9995c71cc0d850b2c83cae2a94fb5  metal_m1u_bec64748_mixtral-8x7b-instruct_default_w256.tsv
+6ac757f867348164f6c4e322a99dc63d11d0715ce94ccdfa1a95b35d15dc7b8d  metal_m1u_bec64748_mixtral-8x7b-instruct_default_w8.tsv
+828ab85e81b83f278cd41d9e0cd589b476e909d6fb781f4453ed73a6103cc18b  metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w1.tsv
+6bc8cd8b028c616c2145ecf1b97dc1c8dae9995c71cc0d850b2c83cae2a94fb5  metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w256.tsv
+6ac757f867348164f6c4e322a99dc63d11d0715ce94ccdfa1a95b35d15dc7b8d  metal_m1u_bec64748_mixtral-8x7b-instruct_fused0_w8.tsv
+f6b828c0e14a3e847265ec81f143c215cb0b4dc8085771cfd91d835f2a78aad5  metal_m1u_bec64748_qwen3-0.6b_default_w1.tsv
+67a496e8856502916e707b84901bf1616fc3b840302834a56627289c9ed1c7e0  metal_m1u_bec64748_qwen3-0.6b_default_w256.tsv
+7b88b45488d90e3ac39312bf786037bf93344f3ebc9c0507fb8c2b9ecacafb70  metal_m1u_bec64748_qwen3-0.6b_default_w8.tsv
+10fa60dc3e6bf31f86ec5a99c93f8dbe010eafa4916863c753d6777b6ccd970c  metal_m1u_bec64748_qwen3-30b-a3b_default_w1.tsv
+a4c3ddeefddd6124f36e2b130437197258bcaf06284063e1d2cc51aeeedded9c  metal_m1u_bec64748_qwen3-30b-a3b_default_w256.tsv
+5140e73dbfb7afb09829d73e718134abdf78c15f79e693eb545e05caa9e5dc65  metal_m1u_bec64748_qwen3-30b-a3b_default_w8.tsv
+aef092dfa146334097190efafd2428b88791de6a3310361691a4381eba48aeda  metal_m1u_bec64748_qwen3-30b-a3b_fused0_w1.tsv
+a4c3ddeefddd6124f36e2b130437197258bcaf06284063e1d2cc51aeeedded9c  metal_m1u_bec64748_qwen3-30b-a3b_fused0_w256.tsv
+5140e73dbfb7afb09829d73e718134abdf78c15f79e693eb545e05caa9e5dc65  metal_m1u_bec64748_qwen3-30b-a3b_fused0_w8.tsv