From 2634fa5088dd954e38068dabe207849a072c8e30 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 29 Sep 2026 07:53:35 -0300 Subject: [PATCH 1/3] Ternary Bonsai on HRX: exact PTQ1_0 -> Q4_0 converter and the prism.hadamard route PrismML's Ternary-Bonsai-2-27B (Qwen3.8-27B trained ternary) now runs on HRX0. - tools/ternary_to_q4_0.py writes PrismML's PTQ1_0 weights as Q4_0, bit for bit (q = trit + 8, the group's fp16 scale in each of its four blocks), decodes every group back before keeping the file, keeps all metadata and stamps onebit.ternary_q4_0 = 128. tests/ternary_to_q4_0_test.py checks it against PrismML's own encoder (run in CI next to the Q4NX repack test). - 1bit serve sends a prism.hadamard file to --device hrx (auto picks it; other devices are refused, upstream llama.cpp would ignore the rotation and answer garbage) and refuses PrismML's own ternary types with the converter's name. tests/prism_route.sh. - docs/hrx.md: how to run it, what was checked, and the one-sequence limit of the Qwen3.5 / Qwen3.8 gated delta net on HRX0. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 3 +- CMakeLists.txt | 1 + app/serve.cpp | 27 ++++ docs/hrx.md | 33 ++++- tests/prism_route.sh | 90 +++++++++++++ tests/ternary_to_q4_0_test.py | 155 ++++++++++++++++++++++ tools/ternary_to_q4_0.py | 234 ++++++++++++++++++++++++++++++++++ 7 files changed, 541 insertions(+), 2 deletions(-) create mode 100755 tests/prism_route.sh create mode 100644 tests/ternary_to_q4_0_test.py create mode 100644 tools/ternary_to_q4_0.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09a45fb3..6f95ab26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,10 +36,11 @@ jobs: run: | git fetch -q --depth=1 origin "$BASE" python3 tools/check_pins.py "$BASE" - - name: GGUF -> Q4NX repack + - name: GGUF -> Q4NX repack, ternary -> Q4_0 run: | python3 -m pip install --user numpy gguf python3 tests/gguf_to_q4nx_test.py + python3 tests/ternary_to_q4_0_test.py - name: Install build dependencies # GCC 15, as on the Strix Halo development machine: the engine is C++26, which the # runner's own GCC 13 cannot build diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b27ff11..5cbe1d08 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,7 @@ add_test(NAME hadamard_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hadamard_route.sh add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $) add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $) add_test(NAME bench_selftest COMMAND ${CMAKE_SOURCE_DIR}/tests/bench_selftest.sh $) +add_test(NAME prism_route COMMAND ${CMAKE_SOURCE_DIR}/tests/prism_route.sh $) add_test(NAME long_route COMMAND ${CMAKE_SOURCE_DIR}/tests/long_route.sh $) add_test(NAME laya_gguf_route COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_gguf_route.sh $) # The NPU route's split into reasoning_content (app/think_split.h); header only, runs in CI. diff --git a/app/serve.cpp b/app/serve.cpp index 663fc5e2..62148bfe 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -212,6 +212,21 @@ bool hadamard_q4_0(const std::string& model) { gguf_int(model, "onebit.hadamard_q4_0") == 32; } +// A PrismML model (Ternary Bonsai): prism.hadamard.* says its weights are stored rotated by a +// Walsh-Hadamard transform. Our llama.cpp (the HRX build, src/llama-hadamard.cpp) rotates the +// activations to match; upstream llama.cpp would load the file and answer garbage. +bool prism_hadamard(const std::string& model) { + return model.size() > 5 && model.compare(model.size() - 5, 5, ".gguf") == 0 && + gguf_int(model, "prism.hadamard.version") > 0; +} + +// PrismML's own ternary types (general.file_type PQ2_0 / PTQ1_0 in its fork): no pinned +// llama.cpp reads them; tools/ternary_to_q4_0.py writes the same weights as Q4_0. +bool prism_ternary_types(const std::string& model) { + const long long ft = gguf_int(model, "general.file_type"); + return ft == 128 || ft == 129 || ft == 141 || ft == 142 || ft == 143; +} + // A 1BP package (1bit-MONSTER's model format; our DwarfStar fork reads version 5, its // docs/1BP.md): its first four bytes are "1BP\0". bool onebp_file(const std::string& model) { @@ -1110,6 +1125,18 @@ int serve_child(const Options& given) { throw std::runtime_error("--laya routes among devices; a Hadamard-rotated file runs on rocm only"); std::fprintf(stderr, "1bit serve: %s is Hadamard-rotated: lean ROCm route, W4A4 prompt processing\n", o.model.c_str()); } + if (prism_hadamard(o.model)) { + if (prism_ternary_types(o.model)) + throw std::runtime_error(o.model + " stores PrismML's ternary types: convert it first with " + "tools/ternary_to_q4_0.py (the same weights as Q4_0), docs/hrx.md"); + // a rotated file has one route: our llama.cpp on HRX, which rotates the activations + if (o.device == "auto") o.device = "hrx"; + if (o.device != "hrx") + throw std::runtime_error(o.model + " is Hadamard-folded (prism.hadamard): it runs on --device hrx only, docs/hrx.md"); + if (o.laya_auto || !o.laya_model.empty()) + throw std::runtime_error("--laya routes among devices; a Hadamard-folded file runs on hrx only"); + std::fprintf(stderr, "1bit serve: %s is Hadamard-folded (prism.hadamard): HRX route\n", o.model.c_str()); + } if (onebp_file(o.model)) { // DwarfStar (our fork) is the engine's 1BP reader if (o.device == "auto") o.device = "ds4"; diff --git a/docs/hrx.md b/docs/hrx.md index 61784767..d588a37e 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -185,10 +185,41 @@ were observed. reference kernel and `check.case` differentials that pass under `iree-test-loom --sanitizer=access`. `GGML_HRX_DISABLE_DISPATCH=zaya.cca` turns the two ZAYA kernels off for comparison. +### Ternary Bonsai (PrismML's Hadamard-folded GGUFs) + +PrismML's [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) is Qwen3.8-27B +trained to weights of -1, 0 and +1, with one fp16 scale per 128 weights. It runs on `HRX0`: + +```sh +tools/ternary_to_q4_0.py Ternary-Bonsai-2-27B-PTQ1_0.gguf Ternary-Bonsai-2-27B-Q4_0.gguf +1bit serve -m Ternary-Bonsai-2-27B-Q4_0.gguf # --device auto picks hrx for this file +``` + +- **The weights.** PrismML's PTQ1_0 and PQ2_0 types are not in any llama.cpp the engine pins. + `tools/ternary_to_q4_0.py` writes the same weights as Q4_0 (q = trit + 8 with the group's own + fp16 scale in each of its four blocks), so nothing is lost: every group is decoded back and + compared before the file is kept, and `tests/ternary_to_q4_0_test.py` checks it against + PrismML's own encoder. The file is 14.1 GiB instead of 5.5; a ternary kernel for HRX would win + that back. +- **The rotation.** Bonsai stores almost every matmul weight rotated by a 1024-point + Walsh-Hadamard transform with fixed signs (`prism.hadamard.*` keys), and the activations have to + be rotated to match. Our llama.cpp does that (`src/llama-hadamard.{h,cpp}`, llama.cpp #34, + following PrismML's implementation): a plain F32 matmul against the rotation matrix before each + folded weight, the inverse after the token-embedding lookup, and a check on the first graph that + no folded weight is used without it. Upstream llama.cpp would load the file and answer garbage, + so `1bit serve` sends a `prism.hadamard` file to `hrx` only, and refuses a file still in + PrismML's own types with the converter's name (`tests/prism_route.sh`). +- **Checked.** Against the F16 model on the CPU in PrismML's own fork, wikitext-2 running + perplexity on `HRX0` is 7.962 / 10.408 / 10.171 at chunks 5 / 10 / 15 (CPU: 7.958 / 10.382 / + 10.142), the difference Q4_0's 8-bit activations make. Files without the keys build the same graph + as before. + ### Known issues on `HRX0` - **Several sequences per batch fail.** `llama-perplexity` with `n_seq` > 1 stops on an - unsupported 3-D MUL_MAT; use `-b 512`. + unsupported 3-D MUL_MAT; use `-b 512`. On Qwen3.5 / Qwen3.8 (and Bonsai) the gated delta net runs + one sequence per batch: with two or more, its fused kernel does not match and the graph stops on + the softplus it would have covered, so `--parallel` above 1 answers HTTP 500 there. - **`-fa off` fails.** A SET_ROWS into the non-flash-attention V cache is rejected. - **Decode-split flash attention is on.** `flash_attention_decode_split_next_q8` used to give nondeterministic attention on HRX0 ([#140](https://github.com/1bit-MONSTER/engine/issues/140)) and diff --git a/tests/prism_route.sh b/tests/prism_route.sh new file mode 100755 index 00000000..91dd3973 --- /dev/null +++ b/tests/prism_route.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# How `1bit serve` routes a PrismML Hadamard-folded file (Ternary Bonsai; prism.hadamard.* keys, +# docs/hrx.md), without a GPU: +# - --device auto sends a converted file (tools/ternary_to_q4_0.py) to HRX0, +# - --device vulkan refuses it (upstream llama.cpp would ignore the rotation), +# - a file still in PrismML's ternary types is refused with the converter's name, +# - a plain file with --device auto still goes to Vulkan0. +# +# usage: tests/prism_route.sh path/to/1bit +set -uo pipefail + +bin=${1:?usage: prism_route.sh path/to/1bit} +scratch=$(mktemp -d) +trap 'kill -9 $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$scratch"' EXIT +pid= +fail=0 +check() { if eval "$2"; then echo "ok $1"; else echo "FAIL $1"; fail=1; fi; } + +python3 - "$scratch" <<'PY' +import struct, sys +def gguf(path, prism, file_type): + kv = [] + def s(x): b = x.encode(); return struct.pack(" "$scratch/backend.py" <<'PY' +#!/usr/bin/env python3 +import json, os, sys +from http.server import BaseHTTPRequestHandler, HTTPServer +argv = sys.argv[1:] +port = int(argv[argv.index("--port") + 1]) +open(os.environ["RECORD"], "w").write(json.dumps({"argv": argv})) +class H(BaseHTTPRequestHandler): + def log_message(self, *a): pass + def do_GET(self): + b = b'{"status":"ok"}' + self.send_response(200); self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b) +HTTPServer(("127.0.0.1", port), H).serve_forever() +PY +chmod +x "$scratch/backend.py" + +refused=$("$bin" serve -m "$scratch/bonsai-q4_0.gguf" --device vulkan --port 1 --llama-server "$scratch/backend.py" 2>&1) +check "--device vulkan refuses a Hadamard-folded file" '[[ "$refused" == *"Hadamard-folded"* ]]' +refused=$("$bin" serve -m "$scratch/bonsai-ptq1_0.gguf" --device auto --port 1 --llama-server "$scratch/backend.py" 2>&1) +check "PrismML ternary types are refused with the converter's name" '[[ "$refused" == *"ternary_to_q4_0.py"* ]]' + +run() { # : serve with --device auto until the backend has recorded its start + local port + port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])') + RECORD="$2" "$bin" serve -m "$1" --device auto --port "$port" --llama-server "$scratch/backend.py" >"$2.log" 2>&1 & + pid=$! + for _ in $(seq 1 100); do [ -s "$2" ] && break; sleep 0.1; done + kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null +} +field() { python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); print(eval(sys.argv[2]))' "$1" "$2"; } + +run "$scratch/bonsai-q4_0.gguf" "$scratch/b.json" +check "--device auto sends a Hadamard-folded file to HRX0" '[ "$(field "$scratch/b.json" "r[\"argv\"][r[\"argv\"].index(\"--device\")+1]")" = HRX0 ]' +run "$scratch/plain.gguf" "$scratch/plain.json" +check "a plain file still goes to Vulkan0" '[ "$(field "$scratch/plain.json" "r[\"argv\"][r[\"argv\"].index(\"--device\")+1]")" = Vulkan0 ]' + +if [ $fail -ne 0 ]; then for f in "$scratch"/*.log; do echo "--- $f"; cat "$f"; done; echo FAIL; exit 1; fi +echo PASS diff --git a/tests/ternary_to_q4_0_test.py b/tests/ternary_to_q4_0_test.py new file mode 100644 index 00000000..f5684e39 --- /dev/null +++ b/tests/ternary_to_q4_0_test.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""tests/ternary_to_q4_0_test.py — tools/ternary_to_q4_0.py on a small made-up PTQ1_0 GGUF. + +The PTQ1_0 blocks are written here with PrismML's own encoder (quantize_row_ptq1_0_ref in its +ggml-quants.c, ported line by line), not with the tool's decoder, and the Q4_0 output is read +back with Q4_0's definition (w = (q - 8) * d): every weight must come back exactly, the other +tensors byte for byte, the metadata unchanged apart from general.file_type and the stamp. +""" +import os +import struct +import subprocess +import sys +import tempfile + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +TOOL = os.path.join(HERE, "..", "tools", "ternary_to_q4_0.py") +STAGES = (32, 16, 8) + + +def ptq1_0_block(x): + """PrismML's quantize_row_ptq1_0_ref for one block of 128 floats""" + d = np.float16(np.abs(x).max()) + idv = 1.0 / float(d) if d else 0.0 + qs, j, off = [0] * 24, 0, 0 + for c in STAGES: + while j + c <= 24: + for m in range(c): + q = 0 + for n in range(5): + q = q * 3 + int(np.round(x[off + m + n * c] * idv)) + 1 + qs[j + m] = (q * 256 + 242) // 243 + off += 5 * c + j += c + qh = [] + for h in range(2): + q = 0 + for m in range(4): + q = q * 3 + int(np.round(x[off + h + m * 2] * idv)) + 1 + q *= 3 + qh.append((q * 256 + 242) // 243) + return bytes(qs) + bytes(qh) + d.tobytes() + + +def s(x): + b = x.encode() + return struct.pack(" (kvs {key: raw value bytes}, tensors {name: (type, dims, bytes)})""" + b = open(path, "rb").read() + assert b[:4] == b"GGUF" + n_t, n_kv = struct.unpack_from("> 4], axis=1).astype(np.float32) - 8 + back = (nib * d[:, None]).reshape(rows, cols) + if t != 2 or dims != (cols, rows): + fail.append(f"ffn_up is type {t} {dims}, want Q4_0 {(cols, rows)}") + if not np.array_equal(back, w): + fail.append(f"Q4_0 weights differ from the PTQ1_0 ones in {int((back != w).sum())} places") + if ot["blk.0.attn_norm.weight"][2] != norm: + fail.append("the F32 tensor changed") + if okv["general.file_type"][1] != struct.pack(" Q4_0 exact, other tensors and metadata kept") + sys.exit(1 if fail else 0) + + +if __name__ == "__main__": + main() diff --git a/tools/ternary_to_q4_0.py b/tools/ternary_to_q4_0.py new file mode 100644 index 00000000..5fa11cb0 --- /dev/null +++ b/tools/ternary_to_q4_0.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""ternary_to_q4_0.py — a ternary GGUF (PrismML PTQ1_0) as Q4_0, bit for bit the same weights. + + tools/ternary_to_q4_0.py IN.gguf OUT.gguf [--check N] + +PTQ1_0 (ggml type 143, PrismML's llama.cpp fork) stores 128 weights as trits t in {-1, 0, +1} +(base-3, 5 per byte) with one fp16 scale d: w = t * d. Q4_0 stores 32 weights as nibbles q with +one fp16 scale: w = (q - 8) * d. Writing q = t + 8 and the same fp16 d into each of the four +Q4_0 blocks of a group gives exactly the same weights, so every backend with a Q4_0 matmul +(HRX among them) runs the model without a ternary kernel. The file is bigger (4.5 instead of +1.75 bits per weight), which is what a native ternary kernel wins back. + +Everything else is copied as it is, metadata included: PrismML's prism.hadamard.* keys say +which weights were stored in a Walsh-Hadamard-rotated basis, and the loader still has to rotate +the activations to match. general.file_type becomes MOSTLY_Q4_0 and onebit.ternary_q4_0 = 128 +(the source group size) marks the file. --check N decodes N random groups per tensor back from +the written file and compares them with the source (all of them are compared by default: 0 = all). +""" +import argparse +import os +import struct +import sys + +import numpy as np + +GGUF_MAGIC = b"GGUF" +F32, F16, Q4_0, BF16, PTQ1_0 = 0, 1, 2, 30, 143 +# bytes per block, elements per block +SIZES = {F32: (4, 1), F16: (2, 1), Q4_0: (18, 32), BF16: (2, 1), PTQ1_0: (28, 128)} +MOSTLY_Q4_0 = 2 +# the order PTQ1_0 stores its 128 values in (ggml-quants.c, dequantize_row_ptq1_0): bytes +# 0..15 give 5 trits each (value n*16 + m from byte m, trit n), bytes 16..23 the next 40, +# the two qh bytes 4 trits each +POW3 = np.array([1, 3, 9, 27, 81, 243], dtype=np.uint16) + + +def read_str(f): + (n,) = struct.unpack(" (version, [(key, type, raw bytes of type+value)], [tensor dicts], alignment, data offset)""" + with open(path, "rb") as f: + if f.read(4) != GGUF_MAGIC: + raise ValueError(f"{path}: not a GGUF file") + (version,) = struct.unpack(" trits (n, 128) int8 in -1..1, fp16 scales (n,) as uint16""" + qs, qh = raw[:, :24].astype(np.uint16), raw[:, 24:26].astype(np.uint16) + d = raw[:, 26:28].copy().view(np.uint16)[:, 0] + + def digits(b, count): # (n, k) bytes -> (n, count, k) trits, trit n of each byte + q = (b[:, None, :] * POW3[:count, None]) & 0xFF + return ((q * 3) >> 8).astype(np.int8) - 1 + + a = digits(qs[:, :16], 5).reshape(len(raw), 80) + b = digits(qs[:, 16:24], 5).reshape(len(raw), 40) + c = digits(qh, 4).reshape(len(raw), 8) + return np.concatenate([a, b, c], axis=1), d + + +def q4_0_from_trits(t, d): + """trits (n, 128), scales (n,) uint16 -> Q4_0 blocks (4n, 18) uint8 with the same weights""" + n = len(t) + q = (t.reshape(n * 4, 32) + 8).astype(np.uint8) + out = np.empty((n * 4, 18), dtype=np.uint8) + out[:, 0:2] = np.repeat(d, 4).view(np.uint8).reshape(-1, 2) + out[:, 2:] = q[:, :16] | (q[:, 16:] << 4) + return out + + +def q4_0_decode(raw): + """Q4_0 blocks (n, 18) -> (values (n, 32) int8 as q - 8, fp16 scales (n,) uint16)""" + d = raw[:, 0:2].copy().view(np.uint16)[:, 0] + qs = raw[:, 2:] + q = np.concatenate([qs & 0x0F, qs >> 4], axis=1).astype(np.int8) - 8 + return q, d + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("src") + ap.add_argument("dst") + ap.add_argument("--check", type=int, default=0, help="groups per tensor to verify after writing (0 = all)") + a = ap.parse_args() + + version, kvs, tensors, alignment, data = parse(a.src) + if not any(t["type"] == PTQ1_0 for t in tensors): + sys.exit(f"{a.src}: no PTQ1_0 tensors") + for t in tensors: + t["out_type"] = Q4_0 if t["type"] == PTQ1_0 else t["type"] + t["out_bytes"] = nbytes(t["out_type"], t["dims"]) + t["in_bytes"] = nbytes(t["type"], t["dims"]) + + # metadata: the source keys, general.file_type -> MOSTLY_Q4_0, plus our stamp + out_kvs = [] + for key, typ, raw in kvs: + if key == "general.file_type": + raw = struct.pack(" Q4_0, {checked} groups verified identical, " + f"{os.path.getsize(a.dst) / 2**30:.2f} GiB") + + +if __name__ == "__main__": + main() From 31545a4833ca0b627a7d1f66c96dac818f504fef Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 30 Sep 2026 06:08:42 -0300 Subject: [PATCH 2/3] Pin llama.cpp b8d587e: load Hadamard-folded weights (llama.cpp #34) acf9c74 -> b8d587e is llama.cpp #34 only: src/llama-hadamard.{h,cpp} reads prism.hadamard.* and rotates the activations before each folded weight, so Ternary Bonsai runs on HRX0. docs/hrx.md: the Qwen3.5 / Qwen3.8 one-sequence note now points at serve's one-slot default and --parallel refusal (#244). Co-Authored-By: Claude Opus 5.5 --- docs/hrx.md | 6 +++--- third_party/llama.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/hrx.md b/docs/hrx.md index d588a37e..da1b1826 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -217,9 +217,9 @@ tools/ternary_to_q4_0.py Ternary-Bonsai-2-27B-PTQ1_0.gguf Ternary-Bonsai-2-27B-Q ### Known issues on `HRX0` - **Several sequences per batch fail.** `llama-perplexity` with `n_seq` > 1 stops on an - unsupported 3-D MUL_MAT; use `-b 512`. On Qwen3.5 / Qwen3.8 (and Bonsai) the gated delta net runs - one sequence per batch: with two or more, its fused kernel does not match and the graph stops on - the softplus it would have covered, so `--parallel` above 1 answers HTTP 500 there. + unsupported 3-D MUL_MAT; use `-b 512`. On Qwen3.5 / Qwen3.8 (Bonsai too) the gated delta net runs + one sequence per batch, so `serve` gives these models one slot on HRX and refuses `--parallel` + (`--parallel N` under "Not yet" below). - **`-fa off` fails.** A SET_ROWS into the non-flash-attention V cache is rejected. - **Decode-split flash attention is on.** `flash_attention_decode_split_next_q8` used to give nondeterministic attention on HRX0 ([#140](https://github.com/1bit-MONSTER/engine/issues/140)) and diff --git a/third_party/llama.cpp b/third_party/llama.cpp index acf9c747..b8d587e0 160000 --- a/third_party/llama.cpp +++ b/third_party/llama.cpp @@ -1 +1 @@ -Subproject commit acf9c747c3a0046981604615db5d3306ea8f4c98 +Subproject commit b8d587e054177241a31c27fb3f1f77893b272e8d From afbfaa8c0b70e40b2243a4c659d095027cf03c2e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Wed, 30 Sep 2026 06:29:19 -0300 Subject: [PATCH 3/3] registry: regenerate for the llama.cpp b8d587e pin Co-Authored-By: Claude Opus 5.5 --- registry/architectures.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/architectures.json b/registry/architectures.json index 32db6269..21847891 100644 --- a/registry/architectures.json +++ b/registry/architectures.json @@ -2,7 +2,7 @@ "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.", "sources": { "llama.cpp (vulkan)": "cecf3ee01d9d99378e98bfea95f51cac714b04b8", - "llama.cpp (hrx)": "acf9c747c3a0046981604615db5d3306ea8f4c98", + "llama.cpp (hrx)": "b8d587e054177241a31c27fb3f1f77893b272e8d", "zinc": "29bc350ac4cbf9f110ec628b9e177ea04ac816fb" }, "counts": {