Ternary Bonsai on HRX: exact PTQ1_0 -> Q4_0 converter, prism.hadamard route, pin llama.cpp b8d587e

.github/workflows/ci.yml

@@ -36,10 +36,11 @@ jobs:
         run: |
           git fetch -q --depth=1 origin "$BASE"
           python3 tools/check_pins.py "$BASE"
-      - name: GGUF -> Q4NX repack
+      - name: GGUF -> Q4NX repack, ternary -> Q4_0
         run: |
           python3 -m pip install --user numpy gguf
           python3 tests/gguf_to_q4nx_test.py
+          python3 tests/ternary_to_q4_0_test.py
       - name: Install build dependencies
         # GCC 15, as on the Strix Halo development machine: the engine is C++26, which the
         # runner's own GCC 13 cannot build

CMakeLists.txt

@@ -78,6 +78,7 @@ add_test(NAME hadamard_route COMMAND ${CMAKE_SOURCE_DIR}/tests/hadamard_route.sh
 add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $<TARGET_FILE:onebit>)
 add_test(NAME recipes_route COMMAND ${CMAKE_SOURCE_DIR}/tests/recipes_route.sh $<TARGET_FILE:onebit>)
 add_test(NAME bench_selftest COMMAND ${CMAKE_SOURCE_DIR}/tests/bench_selftest.sh $<TARGET_FILE:onebit>)
+add_test(NAME prism_route COMMAND ${CMAKE_SOURCE_DIR}/tests/prism_route.sh $<TARGET_FILE:onebit>)
 add_test(NAME long_route COMMAND ${CMAKE_SOURCE_DIR}/tests/long_route.sh $<TARGET_FILE:onebit>)
 add_test(NAME laya_gguf_route COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_gguf_route.sh $<TARGET_FILE:onebit>)
 # The NPU route's <think> split into reasoning_content (app/think_split.h); header only, runs in CI.

app/serve.cpp

@@ -212,6 +212,21 @@ bool hadamard_q4_0(const std::string& model) {
            gguf_int(model, "onebit.hadamard_q4_0") == 32;
 }
 
+// A PrismML model (Ternary Bonsai): prism.hadamard.* says its weights are stored rotated by a
+// Walsh-Hadamard transform. Our llama.cpp (the HRX build, src/llama-hadamard.cpp) rotates the
+// activations to match; upstream llama.cpp would load the file and answer garbage.
+bool prism_hadamard(const std::string& model) {
+    return model.size() > 5 && model.compare(model.size() - 5, 5, ".gguf") == 0 &&
+           gguf_int(model, "prism.hadamard.version") > 0;
+}
+
+// PrismML's own ternary types (general.file_type PQ2_0 / PTQ1_0 in its fork): no pinned
+// llama.cpp reads them; tools/ternary_to_q4_0.py writes the same weights as Q4_0.
+bool prism_ternary_types(const std::string& model) {
+    const long long ft = gguf_int(model, "general.file_type");
+    return ft == 128 || ft == 129 || ft == 141 || ft == 142 || ft == 143;
+}
+
 // A 1BP package (1bit-MONSTER's model format; our DwarfStar fork reads version 5, its
 // docs/1BP.md): its first four bytes are "1BP\0".
 bool onebp_file(const std::string& model) {
@@ -1110,6 +1125,18 @@ int serve_child(const Options& given) {
             throw std::runtime_error("--laya routes among devices; a Hadamard-rotated file runs on rocm only");
         std::fprintf(stderr, "1bit serve: %s is Hadamard-rotated: lean ROCm route, W4A4 prompt processing\n", o.model.c_str());
     }
+    if (prism_hadamard(o.model)) {
+        if (prism_ternary_types(o.model))
+            throw std::runtime_error(o.model + " stores PrismML's ternary types: convert it first with "
+                                     "tools/ternary_to_q4_0.py (the same weights as Q4_0), docs/hrx.md");
+        // a rotated file has one route: our llama.cpp on HRX, which rotates the activations
+        if (o.device == "auto") o.device = "hrx";
+        if (o.device != "hrx")
+            throw std::runtime_error(o.model + " is Hadamard-folded (prism.hadamard): it runs on --device hrx only, docs/hrx.md");
+        if (o.laya_auto || !o.laya_model.empty())
+            throw std::runtime_error("--laya routes among devices; a Hadamard-folded file runs on hrx only");
+        std::fprintf(stderr, "1bit serve: %s is Hadamard-folded (prism.hadamard): HRX route\n", o.model.c_str());
+    }
     if (onebp_file(o.model)) {
         // DwarfStar (our fork) is the engine's 1BP reader
         if (o.device == "auto") o.device = "ds4";

docs/hrx.md

@@ -185,10 +185,41 @@ were observed.
   reference kernel and `check.case` differentials that pass under `iree-test-loom --sanitizer=access`.
   `GGML_HRX_DISABLE_DISPATCH=zaya.cca` turns the two ZAYA kernels off for comparison.
 
+### Ternary Bonsai (PrismML's Hadamard-folded GGUFs)
+
+PrismML's [Ternary-Bonsai-2-27B](https://huggingface.co/prism-ml/Ternary-Bonsai-2-27B-gguf) is Qwen3.8-27B
+trained to weights of -1, 0 and +1, with one fp16 scale per 128 weights. It runs on `HRX0`:
+
+```sh
+tools/ternary_to_q4_0.py Ternary-Bonsai-2-27B-PTQ1_0.gguf Ternary-Bonsai-2-27B-Q4_0.gguf
+1bit serve -m Ternary-Bonsai-2-27B-Q4_0.gguf          # --device auto picks hrx for this file
+```
+
+- **The weights.** PrismML's PTQ1_0 and PQ2_0 types are not in any llama.cpp the engine pins.
+  `tools/ternary_to_q4_0.py` writes the same weights as Q4_0 (q = trit + 8 with the group's own
+  fp16 scale in each of its four blocks), so nothing is lost: every group is decoded back and
+  compared before the file is kept, and `tests/ternary_to_q4_0_test.py` checks it against
+  PrismML's own encoder. The file is 14.1 GiB instead of 5.5; a ternary kernel for HRX would win
+  that back.
+- **The rotation.** Bonsai stores almost every matmul weight rotated by a 1024-point
+  Walsh-Hadamard transform with fixed signs (`prism.hadamard.*` keys), and the activations have to
+  be rotated to match. Our llama.cpp does that (`src/llama-hadamard.{h,cpp}`, llama.cpp #34,
+  following PrismML's implementation): a plain F32 matmul against the rotation matrix before each
+  folded weight, the inverse after the token-embedding lookup, and a check on the first graph that
+  no folded weight is used without it. Upstream llama.cpp would load the file and answer garbage,
+  so `1bit serve` sends a `prism.hadamard` file to `hrx` only, and refuses a file still in
+  PrismML's own types with the converter's name (`tests/prism_route.sh`).
+- **Checked.** Against the F16 model on the CPU in PrismML's own fork, wikitext-2 running
+  perplexity on `HRX0` is 7.962 / 10.408 / 10.171 at chunks 5 / 10 / 15 (CPU: 7.958 / 10.382 /
+  10.142), the difference Q4_0's 8-bit activations make. Files without the keys build the same graph
+  as before.
+
 ### Known issues on `HRX0`
 
 - **Several sequences per batch fail.** `llama-perplexity` with `n_seq` > 1 stops on an
-  unsupported 3-D MUL_MAT; use `-b 512`.
+  unsupported 3-D MUL_MAT; use `-b 512`. On Qwen3.5 / Qwen3.8 (Bonsai too) the gated delta net runs
+  one sequence per batch, so `serve` gives these models one slot on HRX and refuses `--parallel`
+  (`--parallel N` under "Not yet" below).
 - **`-fa off` fails.** A SET_ROWS into the non-flash-attention V cache is rejected.
 - **Decode-split flash attention is on.** `flash_attention_decode_split_next_q8` used to give
   nondeterministic attention on HRX0 ([#140](https://github.com/1bit-MONSTER/engine/issues/140)) and

registry/architectures.json

@@ -2,7 +2,7 @@
  "about": "HF architecture -> GGUF architecture and the backends whose code accepts it. Generated by tools/registry_build.py from the pinned sources; do not edit.",
  "sources": {
   "llama.cpp (vulkan)": "cecf3ee01d9d99378e98bfea95f51cac714b04b8",
-  "llama.cpp (hrx)": "acf9c747c3a0046981604615db5d3306ea8f4c98",
+  "llama.cpp (hrx)": "b8d587e054177241a31c27fb3f1f77893b272e8d",
   "zinc": "29bc350ac4cbf9f110ec628b9e177ea04ac816fb"
  },
  "counts": {

tests/prism_route.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# How `1bit serve` routes a PrismML Hadamard-folded file (Ternary Bonsai; prism.hadamard.* keys,
+# docs/hrx.md), without a GPU:
+#   - --device auto sends a converted file (tools/ternary_to_q4_0.py) to HRX0,
+#   - --device vulkan refuses it (upstream llama.cpp would ignore the rotation),
+#   - a file still in PrismML's ternary types is refused with the converter's name,
+#   - a plain file with --device auto still goes to Vulkan0.
+#
+# usage: tests/prism_route.sh path/to/1bit
+set -uo pipefail
+
+bin=${1:?usage: prism_route.sh path/to/1bit}
+scratch=$(mktemp -d)
+trap 'kill -9 $pid 2>/dev/null; wait $pid 2>/dev/null; rm -rf "$scratch"' EXIT
+pid=
+fail=0
+check() { if eval "$2"; then echo "ok   $1"; else echo "FAIL $1"; fail=1; fi; }
+
+python3 - "$scratch" <<'PY'
+import struct, sys
+def gguf(path, prism, file_type):
+    kv = []
+    def s(x): b = x.encode(); return struct.pack("<Q", len(b)) + b
+    kv.append(s("general.architecture") + struct.pack("<I", 8) + s("qwen35"))
+    kv.append(s("general.file_type") + struct.pack("<I", 4) + struct.pack("<I", file_type))
+    if prism:
+        names = [b"output.weight", b"blk.0.ffn_up.weight"]
+        kv.append(s("prism.hadamard.weight_names") + struct.pack("<I", 9) + struct.pack("<IQ", 8, len(names))
+                  + b"".join(struct.pack("<Q", len(n)) + n for n in names))
+        kv.append(s("prism.hadamard.version") + struct.pack("<I", 4) + struct.pack("<I", 1))
+    open(path, "wb").write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(kv)) + b"".join(kv))
+gguf(sys.argv[1] + "/bonsai-q4_0.gguf", True, 2)
+gguf(sys.argv[1] + "/bonsai-ptq1_0.gguf", True, 143)
+gguf(sys.argv[1] + "/plain.gguf", False, 2)
+PY
+
+# a backend that records how it was started, then answers /health
+cat > "$scratch/backend.py" <<'PY'
+#!/usr/bin/env python3
+import json, os, sys
+from http.server import BaseHTTPRequestHandler, HTTPServer
+argv = sys.argv[1:]
+port = int(argv[argv.index("--port") + 1])
+open(os.environ["RECORD"], "w").write(json.dumps({"argv": argv}))
+class H(BaseHTTPRequestHandler):
+    def log_message(self, *a): pass
+    def do_GET(self):
+        b = b'{"status":"ok"}'
+        self.send_response(200); self.send_header("Content-Length", str(len(b))); self.end_headers(); self.wfile.write(b)
+HTTPServer(("127.0.0.1", port), H).serve_forever()
+PY
+chmod +x "$scratch/backend.py"
+
+refused=$("$bin" serve -m "$scratch/bonsai-q4_0.gguf" --device vulkan --port 1 --llama-server "$scratch/backend.py" 2>&1)
+check "--device vulkan refuses a Hadamard-folded file" '[[ "$refused" == *"Hadamard-folded"* ]]'
+refused=$("$bin" serve -m "$scratch/bonsai-ptq1_0.gguf" --device auto --port 1 --llama-server "$scratch/backend.py" 2>&1)
+check "PrismML ternary types are refused with the converter's name" '[[ "$refused" == *"ternary_to_q4_0.py"* ]]'
+
+run() {  # <model> <record>: serve with --device auto until the backend has recorded its start
+    local port
+    port=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1])')
+    RECORD="$2" "$bin" serve -m "$1" --device auto --port "$port" --llama-server "$scratch/backend.py" >"$2.log" 2>&1 &
+    pid=$!
+    for _ in $(seq 1 100); do [ -s "$2" ] && break; sleep 0.1; done
+    kill -9 "$pid" 2>/dev/null; wait "$pid" 2>/dev/null
+}
+field() { python3 -c 'import json,sys; r=json.load(open(sys.argv[1])); print(eval(sys.argv[2]))' "$1" "$2"; }
+
+run "$scratch/bonsai-q4_0.gguf" "$scratch/b.json"
+check "--device auto sends a Hadamard-folded file to HRX0" '[ "$(field "$scratch/b.json" "r[\"argv\"][r[\"argv\"].index(\"--device\")+1]")" = HRX0 ]'
+run "$scratch/plain.gguf" "$scratch/plain.json"
+check "a plain file still goes to Vulkan0" '[ "$(field "$scratch/plain.json" "r[\"argv\"][r[\"argv\"].index(\"--device\")+1]")" = Vulkan0 ]'
+
+if [ $fail -ne 0 ]; then for f in "$scratch"/*.log; do echo "--- $f"; cat "$f"; done; echo FAIL; exit 1; fi
+echo PASS

tests/ternary_to_q4_0_test.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""tests/ternary_to_q4_0_test.py — tools/ternary_to_q4_0.py on a small made-up PTQ1_0 GGUF.
+
+The PTQ1_0 blocks are written here with PrismML's own encoder (quantize_row_ptq1_0_ref in its
+ggml-quants.c, ported line by line), not with the tool's decoder, and the Q4_0 output is read
+back with Q4_0's definition (w = (q - 8) * d): every weight must come back exactly, the other
+tensors byte for byte, the metadata unchanged apart from general.file_type and the stamp.
+"""
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+
+import numpy as np
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+TOOL = os.path.join(HERE, "..", "tools", "ternary_to_q4_0.py")
+STAGES = (32, 16, 8)
+
+
+def ptq1_0_block(x):
+    """PrismML's quantize_row_ptq1_0_ref for one block of 128 floats"""
+    d = np.float16(np.abs(x).max())
+    idv = 1.0 / float(d) if d else 0.0
+    qs, j, off = [0] * 24, 0, 0
+    for c in STAGES:
+        while j + c <= 24:
+            for m in range(c):
+                q = 0
+                for n in range(5):
+                    q = q * 3 + int(np.round(x[off + m + n * c] * idv)) + 1
+                qs[j + m] = (q * 256 + 242) // 243
+            off += 5 * c
+            j += c
+    qh = []
+    for h in range(2):
+        q = 0
+        for m in range(4):
+            q = q * 3 + int(np.round(x[off + h + m * 2] * idv)) + 1
+        q *= 3
+        qh.append((q * 256 + 242) // 243)
+    return bytes(qs) + bytes(qh) + d.tobytes()
+
+
+def s(x):
+    b = x.encode()
+    return struct.pack("<Q", len(b)) + b
+
+
+def kv_u32(k, v):
+    return s(k) + struct.pack("<II", 4, v)
+
+
+def read(path):
+    """-> (kvs {key: raw value bytes}, tensors {name: (type, dims, bytes)})"""
+    b = open(path, "rb").read()
+    assert b[:4] == b"GGUF"
+    n_t, n_kv = struct.unpack_from("<QQ", b, 8)
+    p, kvs = 24, {}
+    for _ in range(n_kv):
+        (n,) = struct.unpack_from("<Q", b, p); k = b[p + 8:p + 8 + n].decode(); p += 8 + n
+        (t,) = struct.unpack_from("<I", b, p); p += 4
+        start = p
+        if t == 4:
+            p += 4
+        elif t == 8:
+            (n,) = struct.unpack_from("<Q", b, p); p += 8 + n
+        elif t == 9:
+            et, cnt = struct.unpack_from("<IQ", b, p); p += 12
+            for _ in range(cnt):
+                (n,) = struct.unpack_from("<Q", b, p); p += 8 + n
+        kvs[k] = (t, b[start:p])
+    infos = []
+    for _ in range(n_t):
+        (n,) = struct.unpack_from("<Q", b, p); name = b[p + 8:p + 8 + n].decode(); p += 8 + n
+        (nd,) = struct.unpack_from("<I", b, p); p += 4
+        dims = struct.unpack_from(f"<{nd}Q", b, p); p += 8 * nd
+        t, off = struct.unpack_from("<IQ", b, p); p += 12
+        infos.append((name, t, dims, off))
+    data = (p + 31) // 32 * 32
+    size = {0: lambda n: 4 * n, 2: lambda n: n // 32 * 18, 143: lambda n: n // 128 * 28}
+    return kvs, {nm: (t, dims, b[data + off:data + off + size[t](int(np.prod(dims)))]) for nm, t, dims, off in infos}
+
+
+def main():
+    rng = np.random.default_rng(7)
+    rows, cols = 3, 256
+    trits = rng.integers(-1, 2, size=(rows, cols)).astype(np.float32)
+    scales = rng.uniform(0.001, 0.05, size=(rows, cols // 128)).astype(np.float16).astype(np.float32)
+    trits[0, :128] = 0  # an all-zero group: scale 0
+    w = trits * np.repeat(scales, 128, axis=1)
+    w[0, :128] = 0
+    ptq = b"".join(ptq1_0_block(w[r, g * 128:(g + 1) * 128]) for r in range(rows) for g in range(cols // 128))
+    norm = rng.standard_normal(cols).astype(np.float32).tobytes()
+
+    kvs = [s("general.architecture") + struct.pack("<I", 8) + s("qwen35"), kv_u32("general.file_type", 143),
+           s("prism.hadamard.weight_names") + struct.pack("<IIQ", 9, 8, 1) + s("blk.0.ffn_up.weight"),
+           kv_u32("prism.hadamard.version", 1)]
+    tensors = [("blk.0.ffn_up.weight", 143, (cols, rows), ptq), ("blk.0.attn_norm.weight", 0, (cols,), norm)]
+    head = b"GGUF" + struct.pack("<IQQ", 3, len(tensors), len(kvs)) + b"".join(kvs)
+    off, blobs = 0, b""
+    for name, t, dims, blob in tensors:
+        head += s(name) + struct.pack("<I", len(dims)) + struct.pack(f"<{len(dims)}Q", *dims) + struct.pack("<IQ", t, off)
+        pad = (-len(blob)) % 32
+        blobs += blob + b"\0" * pad
+        off += len(blob) + pad
+    head += b"\0" * ((-len(head)) % 32)
+
+    with tempfile.TemporaryDirectory() as tmp:
+        src, dst = os.path.join(tmp, "t.gguf"), os.path.join(tmp, "q.gguf")
+        open(src, "wb").write(head + blobs)
+        subprocess.run([sys.executable, TOOL, src, dst], check=True)
+        okv, ot = read(dst)
+
+    fail = []
+    t, dims, blob = ot["blk.0.ffn_up.weight"]
+    q = np.frombuffer(blob, np.uint8).reshape(-1, 18)
+    d = q[:, :2].copy().view(np.float16)[:, 0].astype(np.float32)
+    nib = np.concatenate([q[:, 2:] & 15, q[:, 2:] >> 4], axis=1).astype(np.float32) - 8
+    back = (nib * d[:, None]).reshape(rows, cols)
+    if t != 2 or dims != (cols, rows):
+        fail.append(f"ffn_up is type {t} {dims}, want Q4_0 {(cols, rows)}")
+    if not np.array_equal(back, w):
+        fail.append(f"Q4_0 weights differ from the PTQ1_0 ones in {int((back != w).sum())} places")
+    if ot["blk.0.attn_norm.weight"][2] != norm:
+        fail.append("the F32 tensor changed")
+    if okv["general.file_type"][1] != struct.pack("<I", 2):
+        fail.append("general.file_type is not MOSTLY_Q4_0")
+    if okv.get("onebit.ternary_q4_0", (0, b""))[1] != struct.pack("<I", 128):
+        fail.append("no onebit.ternary_q4_0 = 128 stamp")
+    if "prism.hadamard.weight_names" not in okv or "prism.hadamard.version" not in okv:
+        fail.append("prism.hadamard keys were dropped")
+    for f in fail:
+        print("FAIL", f)
+    print("FAIL" if fail else "PASS: PTQ1_0 -> Q4_0 exact, other tensors and metadata kept")
+    sys.exit(1 if fail else 0)
+
+
+if __name__ == "__main__":
+    main()