From 1c15f38a903022a2924dfe2994cded765a4c73d0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 27 Sep 2026 14:55:03 -0300 Subject: [PATCH] hrx: order the decode-split q8 pack after the reduce's global output stores (engine#123, #140) Every decode-split reduce_fused variant (direct, cooperative, multipass, in both the ggml.* and qwen3_moe corpora) writes the reduced attention output to global memory and then packs it into next_q8 with pack_completed_q8, which reads that output back with a different wave/lane mapping. The barrier between them was kernel.barrier, which orders workgroup (LDS) memory only; it compiled to s_waitcnt lgkmcnt(0); s_barrier, with no store-completion wait before it and no buffer_gl0_inv after it. A pack wave could therefore read output before another wave's stores landed, or from a stale L0 line (WGP mode, or after a CWSR restore onto another CU), and quantize the arena slot's previous contents. The f32 output stays correct but next_q8, which the following matmul consumes, is wrong, and NaN when the stale bytes decode as NaN. That is the source of the NaN router logits behind engine#123 and of the run-to-run divergence in engine#140. Queue preemption from any process's KFD eviction opens the window, which is why both issues tracked page migration and GPU contention. Use kernel.barrier scope(workgroup) ordering(acq_rel) at the five sites. Measured on gfx1151 with a separate process that only forces KFD queue evictions: - Qwen3-0.6B, 24-token greedy: 0 of 354 divergent (before: 13-34 per ~177) - Qwen3-0.6B, forced compaction + mlockall: 0 of 118 (before: 42-62 %) - Qwen3-Coder-30B, 2113-token prompt, GGML_HRX_FA_PARTIAL_ALIGN=256: 16 of 16 correct, 0 NaN (before: 1 of 8 correct, NaN guard fired 7 times) Co-Authored-By: Claude Opus 5.5 --- .../ops/flash_attention_decode_split_f32_f16_wmma.loom | 6 +++--- .../flash_attention_decode_split_f32_f16_wmma.loom | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom index f078e4eeb96d..2e0db9f3b851 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom @@ -967,7 +967,7 @@ template.def<@ggml.flash_attention.decode_split.reduce_fused> device priority(20 kernel.barrier scope(workgroup) ordering(acquire) template.apply<@ggml.flash_attention.decode_split.reduce_completed.direct>(%partial_block_capacity, %producer_block_count, %partial_max_noalias, %partial_sum_noalias, %partial_output_noalias, %output_noalias) : (index, index, buffer, buffer, buffer, buffer) scf.if %publish_q8 { - kernel.barrier scope(workgroup) ordering(acq_rel) + kernel.barrier scope(workgroup) ordering(acq_rel) template.apply<@ggml.flash_attention.decode_split.pack_completed_q8>(%key_value_head, %output_noalias, %q8_output) : (index, buffer, buffer) } // Do not expose the reset until every final F32 and Q8 store completes. @@ -1015,7 +1015,7 @@ template.def<@ggml.flash_attention.decode_split.reduce_fused> device priority(10 kernel.barrier scope(workgroup) ordering(acquire) template.apply<@ggml.flash_attention.decode_split.reduce_completed.cooperative>(%partial_block_capacity, %producer_block_count, %partial_max_noalias, %partial_sum_noalias, %partial_output_noalias, %output_noalias) : (index, index, buffer, buffer, buffer, buffer) scf.if %publish_q8 { - kernel.barrier scope(workgroup) ordering(acq_rel) + kernel.barrier scope(workgroup) ordering(acq_rel) template.apply<@ggml.flash_attention.decode_split.pack_completed_q8>(%key_value_head, %output_noalias, %q8_output) : (index, buffer, buffer) } // Do not expose the reset until every final F32 and Q8 store completes. @@ -1063,7 +1063,7 @@ template.def<@ggml.flash_attention.decode_split.reduce_fused> device priority(5) kernel.barrier scope(workgroup) ordering(acquire) template.apply<@ggml.flash_attention.decode_split.reduce_completed.multipass>(%partial_block_capacity, %producer_block_count, %partial_max_noalias, %partial_sum_noalias, %partial_output_noalias, %output_noalias) : (index, index, buffer, buffer, buffer, buffer) scf.if %publish_q8 { - kernel.barrier scope(workgroup) ordering(acq_rel) + kernel.barrier scope(workgroup) ordering(acq_rel) template.apply<@ggml.flash_attention.decode_split.pack_completed_q8>(%key_value_head, %output_noalias, %q8_output) : (index, buffer, buffer) } kernel.barrier scope(workgroup) ordering(release) diff --git a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom index 7c4c6a437537..cf8184a67698 100644 --- a/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom +++ b/ggml/src/ggml-hrx/kernel-corpus/kernels/qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom @@ -829,7 +829,7 @@ template.def<@qwen3_moe.attention.decode_split.reduce_fused> device priority(20) kernel.barrier scope(workgroup) ordering(acquire) template.apply<@qwen3_moe.attention.decode_split.reduce_completed.direct>(%partial_block_capacity, %producer_block_count, %partial_max_noalias, %partial_sum_noalias, %partial_output_noalias, %output_noalias) : (index, index, buffer, buffer, buffer, buffer) scf.if %publish_q8 { - kernel.barrier scope(workgroup) ordering(acq_rel) + kernel.barrier scope(workgroup) ordering(acq_rel) template.apply<@qwen3_moe.attention.decode_split.pack_completed_q8>(%key_value_head, %output_noalias, %q8_output) : (index, buffer, buffer) } // Do not expose the reset until every final F32 and Q8 store completes. @@ -877,7 +877,7 @@ template.def<@qwen3_moe.attention.decode_split.reduce_fused> device priority(10) kernel.barrier scope(workgroup) ordering(acquire) template.apply<@qwen3_moe.attention.decode_split.reduce_completed.cooperative>(%partial_block_capacity, %producer_block_count, %partial_max_noalias, %partial_sum_noalias, %partial_output_noalias, %output_noalias) : (index, index, buffer, buffer, buffer, buffer) scf.if %publish_q8 { - kernel.barrier scope(workgroup) ordering(acq_rel) + kernel.barrier scope(workgroup) ordering(acq_rel) template.apply<@qwen3_moe.attention.decode_split.pack_completed_q8>(%key_value_head, %output_noalias, %q8_output) : (index, buffer, buffer) } // Do not expose the reset until every final F32 and Q8 store completes.