npu: GGUF-on-NPU serving route (Qwen2.5-7B, MiniCPM4-8B, MiniCPM5-1B answer ' Paris' on the NPU)

CMakeLists.txt

@@ -328,6 +328,11 @@ add_test(NAME npu_full_elf_md5 COMMAND npu_full_elf_test)
 set(ONEBIT_NPU_CAPTURED "" CACHE PATH "Directory of captured layer_ctx<N>.elf instruction ELFs")
 set(ONEBIT_NPU_PDI "" CACHE FILEPATH "The layer kernel's design PDI")
 set(ONEBIT_NPU_GOLDEN "" CACHE PATH "Directory of full ELFs the fast lane ran")
+set(ONEBIT_NPU_KERNELS_DIR "" CACHE PATH "Full ELF / xclbin directory the GGUF-on-NPU forward loads (docs/npu.md)")
+target_compile_definitions(onebit PRIVATE ONEBIT_NPU_REPACK_SCRIPT="${CMAKE_SOURCE_DIR}/scripts/repack_gguf.py")
+if(ONEBIT_NPU_KERNELS_DIR)
+    target_compile_definitions(onebit PRIVATE ONEBIT_NPU_KERNELS_DIR="${ONEBIT_NPU_KERNELS_DIR}")
+endif()
 if(ONEBIT_NPU_CAPTURED AND ONEBIT_NPU_PDI AND ONEBIT_NPU_GOLDEN)
     add_test(NAME npu_full_elf_model
              COMMAND npu_full_elf_test ${ONEBIT_NPU_CAPTURED} ${ONEBIT_NPU_PDI} ${ONEBIT_NPU_GOLDEN})
@@ -373,6 +378,22 @@ if(ONEBIT_NPU)
     target_link_libraries(onebit PRIVATE onebit_npu_lane)
     target_compile_definitions(onebit PRIVATE ONEBIT_NPU)
     set_target_properties(onebit PROPERTIES BUILD_RPATH ${ONEBIT_XRT_ROOT}/lib INSTALL_RPATH ${ONEBIT_XRT_ROOT}/lib)
+
+    # The model-generic forward (npu/forward): the GGUF-on-NPU serve route
+    # (docs/npu.md, "Open"). q4nx_forward.cpp + its Q4NX dequant/pack/semantics
+    # + model loader, against the engine tree's I8Ctx header.
+    add_library(onebit_npu_forward STATIC
+        npu/forward/src/model.c
+        npu/forward/src/q4nx_dequant.c
+        npu/forward/src/q4nx_pack.c
+        npu/forward/src/q4nx_semantics.c
+        npu/forward/src/q4nx_forward.cpp)
+    target_include_directories(onebit_npu_forward SYSTEM PUBLIC
+        npu/forward/include npu/forward/engine-src ${ONEBIT_XRT_ROOT}/include)
+    target_link_libraries(onebit_npu_forward PUBLIC
+        ${ONEBIT_XRT_ROOT}/lib/libxrt_coreutil.so aiebu uuid dl pthread m)
+    target_sources(onebit PRIVATE app/forward_serve.cpp)
+    target_link_libraries(onebit PRIVATE onebit_npu_forward)
     set(ONEBIT_NPU_KERNELS "" CACHE PATH "Kernel directory (layer_ctx{1,2,17}.elf, lmhead.elf, layer.pdi)")
     set(ONEBIT_NPU_REFERENCE "" CACHE PATH "Reference lane's logits for the anchor prompt")
     if(ONEBIT_NPU_MODEL AND ONEBIT_NPU_KERNELS AND ONEBIT_NPU_REFERENCE)

app/forward_serve.cpp

@@ -0,0 +1,308 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// `1bit forward-serve -m <model.q4nx> --kernels <elf dir> -p <port>`: one
+// repacked Q4NX model on the model-generic NPU forward, behind OpenAI
+// endpoints. It is the serve route for `1bit serve -m <model.gguf> --device
+// npu` after the GGUF has been repacked (serve.cpp).
+//
+//   GET  /health, /v1/health    200 once the model is on the device
+//   GET  /v1/models             the one model
+//   POST /v1/chat/completions   ChatML prompt, greedy decoding, optional SSE stream
+//   POST /v1/completions        raw prompt
+//
+// Requests run one at a time: the forward holds one KV cache, and each request
+// resets it from position 0.
+#include "forward_serve.h"
+
+#include "../npu/forward/include/q4nx_forward.h"
+#include "tokenizer.h"
+#include <xrt/xrt_device.h>
+
+#include <httplib.h>
+#include <nlohmann/json.hpp>
+
+#include <atomic>
+#include <chrono>
+#include <cstdio>
+#include <filesystem>
+#include <fstream>
+#include <mutex>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace onebit {
+
+namespace {
+
+using json = nlohmann::json;
+
+struct ForwardEngine {
+    std::string id;
+    std::unique_ptr<npu::Tokenizer> tok;
+    int  bos_id  = -1;     // config.json bos_token_id (the chat template may open with it)
+    bool add_bos = false;  // config.json add_bos_token: the rendered chat prompt needs BOS
+    std::unique_ptr<Q4nxNpuForward> fwd;
+    int eos = -1;
+    int endoftext = -1;
+    std::mutex mu;
+};
+
+std::string chatml(const json& messages, bool thinking = true) {
+    std::string p;
+    for (const auto& m : messages) {
+        std::string content;
+        const auto& c = m.at("content");
+        if (c.is_string()) {
+            content = c;
+        } else if (c.is_array()) {
+            for (const auto& part : c)
+                if (part.value("type", "") == "text") content += part.value("text", "");
+        }
+        p += "<|im_start|>" + m.at("role").get<std::string>() + "\n" + content + "<|im_end|>\n";
+    }
+    p += "<|im_start|>assistant\n";
+    (void)thinking;  // the <think> block is a Qwen3-lane idiom; the model-generic
+    // route serves arbitrary archs, whose tokenizers do not know it.
+    return p;
+}
+
+size_t complete_utf8(const std::string& s) {
+    size_t lead = s.size();
+    while (lead > 0 && s.size() - lead < 4 && (static_cast<unsigned char>(s[lead - 1]) & 0xC0) == 0x80) --lead;
+    if (lead == 0) return s.size();
+    const unsigned char c = static_cast<unsigned char>(s[lead - 1]);
+    const size_t need = (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xE ? 3 : (c >> 3) == 0x1E ? 4 : 1;
+    return s.size() - (lead - 1) >= need ? s.size() : lead - 1;
+}
+
+std::string now_id(const char* prefix) {
+    static std::atomic<unsigned> n{0};
+    return std::string(prefix) + std::to_string(std::chrono::system_clock::now().time_since_epoch().count()) + "-" +
+           std::to_string(n++);
+}
+
+int json_int(const std::string& path, const char* key, int def) {
+    std::ifstream f(path);
+    if (!f) return def;
+    json j;
+    f >> j;
+    if (j.contains(key) && j[key].is_number_integer()) return j[key].get<int>();
+    return def;
+}
+
+void handle_generate(ForwardEngine& e, const httplib::Request& req, httplib::Response& res, bool chat) {
+    json body;
+    try {
+        body = json::parse(req.body);
+    } catch (const std::exception&) {
+        res.status = 400;
+        res.set_content(R"({"error":{"message":"request body is not JSON"}})", "application/json");
+        return;
+    }
+    std::string prompt;
+    try {
+        bool thinking = true;
+        if (body.contains("chat_template_kwargs") && body["chat_template_kwargs"].is_object())
+            thinking = body["chat_template_kwargs"].value("enable_thinking", true);
+        prompt = chat ? chatml(body.at("messages"), thinking) : body.at("prompt").get<std::string>();
+    } catch (const std::exception& ex) {
+        res.status = 400;
+        res.set_content(json{{"error", {{"message", std::string("bad request: ") + ex.what()}}}}.dump(), "application/json");
+        return;
+    }
+    const int max_tokens = body.value("max_completion_tokens", body.value("max_tokens", 256));
+    const bool stream = body.value("stream", false);
+    const std::string id = now_id(chat ? "chatcmpl-" : "cmpl-");
+    const long created = long(std::chrono::duration_cast<std::chrono::seconds>(
+                                  std::chrono::system_clock::now().time_since_epoch()).count());
+    const std::string object = chat ? (stream ? "chat.completion.chunk" : "chat.completion") : "text_completion";
+
+    auto run = [&e, prompt, max_tokens, chat](const std::function<bool(const std::string&)>& on_text) {
+        std::lock_guard<std::mutex> lock(e.mu);
+        std::vector<int> ids = e.tok->encode(prompt);
+        // A chat template that opens with {{- bos_token }} (MiniCPM5 and other
+        // llama-class models) needs the BOS id: the ChatML prompt built above never
+        // carries it, and those models derail into their special tokens without it.
+        if (chat && e.add_bos && e.bos_id >= 0) ids.insert(ids.begin(), e.bos_id);
+        if (ids.empty()) throw std::runtime_error("empty prompt");
+
+        auto argmax = [](const std::vector<float>& v) {
+            int a = 0;
+            for (size_t i = 1; i < v.size(); i++)
+                if (v[i] > v[a]) a = (int)i;
+            return a;
+        };
+
+        std::vector<float> logits;
+        for (int i = 0; i < (int)ids.size(); i++) {
+            if (!e.fwd->step(ids[i], i, logits))
+                throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error());
+        }
+
+        std::string pending;
+        bool stopped_at_eos = false;
+        for (int n = 0; n < max_tokens; n++) {
+            const int next = argmax(logits);
+            if (next == e.eos || (e.endoftext >= 0 && next == e.endoftext)) { stopped_at_eos = true; break; }
+            pending += e.tok->decode(next);
+            const size_t cut = complete_utf8(pending);
+            if (cut > 0) {
+                const std::string out = pending.substr(0, cut);
+                pending.erase(0, cut);
+                if (!on_text(out)) break;
+            }
+            if (!e.fwd->step(next, (int)ids.size() + n, logits))
+                throw std::runtime_error(std::string("forward step failed: ") + e.fwd->last_error());
+        }
+        (void)stopped_at_eos;
+    };
+
+    auto wrap = [&](const std::string& content, const std::string& finish) {
+        json out;
+        out["id"] = id;
+        out["object"] = object;
+        out["created"] = created;
+        out["model"] = e.id;
+        if (chat) {
+            out["choices"] = json::array({{{"index", 0},
+                                           {"message", {{"role", "assistant"}, {"content", content}}},
+                                           {"finish_reason", finish}}});
+        } else {
+            out["choices"] = json::array({{{"index", 0}, {"text", content}, {"finish_reason", finish}}});
+        }
+        return out.dump();
+    };
+
+    try {
+        if (!stream) {
+            std::string content;
+            run([&](const std::string& t) {
+                content += t;
+                return true;
+            });
+            res.set_content(wrap(content, "stop"), "application/json");
+        } else {
+            res.set_chunked_content_provider("text/event-stream", [&](size_t, httplib::DataSink& sink) {
+                run([&](const std::string& t) {
+                    json chunk;
+                    chunk["id"] = id;
+                    chunk["object"] = "chat.completion.chunk";
+                    chunk["created"] = created;
+                    chunk["model"] = e.id;
+                    if (chat)
+                        chunk["choices"] = json::array({{{"index", 0},
+                                                         {"delta", {{"content", t}}},
+                                                         {"finish_reason", nullptr}}});
+                    else
+                        chunk["choices"] = json::array({{{"index", 0}, {"text", t}, {"finish_reason", nullptr}}});
+                    const std::string d = "data: " + chunk.dump() + "\n\n";
+                    return sink.write(d.data(), d.size());
+                });
+                const std::string done = "data: [DONE]\n\n";
+                sink.write(done.data(), done.size());
+                sink.done();
+                return true;
+            });
+        }
+    } catch (const std::exception& ex) {
+        res.status = 500;
+        res.set_content(json{{"error", {{"message", ex.what()}}}}.dump(), "application/json");
+    }
+}
+
+}  // namespace
+
+int run_forward_serve(int argc, char** argv) {
+    std::string model, kernels, host = "127.0.0.1", alias;
+    int port = 8000;
+    for (int i = 0; i < argc; i++) {
+        const std::string a = argv[i];
+        auto next = [&]() -> std::string {
+            if (i + 1 >= argc) throw std::runtime_error(a + " needs a value");
+            return argv[++i];
+        };
+        if (a == "-m" || a == "--model") model = next();
+        else if (a == "--kernels") kernels = next();
+        else if (a == "-p" || a == "--port") port = std::stoi(next());
+        else if (a == "--host") host = next();
+        else if (a == "--alias") alias = next();
+        else throw std::runtime_error("unknown option " + a);
+    }
+    if (model.empty()) throw std::runtime_error("usage: 1bit forward-serve -m <model.q4nx> --kernels <dir> [-p PORT]");
+    if (!std::filesystem::exists(model)) throw std::runtime_error(model + " does not exist");
+
+    const std::string dir = std::filesystem::path(model).parent_path().string();
+    const std::string cfg = dir + "/config.json";
+    const std::string tok_json = dir + "/tokenizer.json";
+
+    ForwardEngine e;
+    e.id = alias.empty() ? std::filesystem::path(dir).filename().string() : alias;
+    e.tok = std::make_unique<npu::Tokenizer>(tok_json);
+    e.bos_id  = json_int(cfg, "bos_token_id", -1);
+    e.add_bos = json_int(cfg, "add_bos_token", 0) != 0;
+    e.eos = json_int(cfg, "eos_token_id", -1);
+    e.endoftext = e.tok->token_id("<|endoftext|>");
+
+    e.fwd = std::make_unique<Q4nxNpuForward>();
+    xrt::device dev(0);
+    if (!e.fwd->init(dev, model.c_str(), kernels, /*use_elf=*/true))
+        throw std::runtime_error(std::string("forward init failed: ") + e.fwd->last_error());
+
+    std::atomic<bool> ready{false};
+    httplib::Server srv;
+    srv.Get("/health", [&](const httplib::Request&, httplib::Response& r) {
+        r.status = ready ? 200 : 503;
+        r.set_content(ready ? R"({"status":"ok"})" : R"({"status":"loading"})", "application/json");
+    });
+    srv.Get("/v1/health", [&](const httplib::Request&, httplib::Response& r) {
+        r.status = ready ? 200 : 503;
+        r.set_content(ready ? R"({"status":"ok"})" : R"({"status":"loading"})", "application/json");
+    });
+    srv.Get("/v1/models", [&](const httplib::Request&, httplib::Response& r) {
+        r.set_content(json{{"object", "list"},
+                           {"data", json::array({{{"id", e.id}, {"object", "model"}, {"owned_by", "1bit"}}})}}.dump(),
+                      "application/json");
+    });
+    auto post = [&](const httplib::Request& q, httplib::Response& r, bool chat) {
+        if (!ready) {
+            r.status = 503;
+            r.set_content(R"({"error":{"message":"model is loading"}})", "application/json");
+            return;
+        }
+        handle_generate(e, q, r, chat);
+    };
+    srv.Post("/v1/chat/completions", [&](const httplib::Request& q, httplib::Response& r) { post(q, r, true); });
+    srv.Post("/v1/completions", [&](const httplib::Request& q, httplib::Response& r) { post(q, r, false); });
+
+    if (!srv.bind_to_port(host, port)) throw std::runtime_error("cannot listen on " + host + ":" + std::to_string(port));
+    std::thread listener([&] { srv.listen_after_bind(); });
+    struct Stop {
+        httplib::Server& s;
+        std::thread& t;
+        ~Stop() { s.stop(); if (t.joinable()) t.join(); }
+    } stop{srv, listener};
+
+    ready = true;
+    std::fprintf(stderr, "1bit forward-serve: %s ready on http://%s:%d (%s)\n",
+                 e.id.c_str(), host.c_str(), port, e.fwd->backend().c_str());
+
+    while (true) std::this_thread::sleep_for(std::chrono::seconds(3600));
+    return 0;
+}
+
+}  // namespace onebit

app/forward_serve.h

@@ -0,0 +1,26 @@
+// Copyright 2026 bong-water-water-bong
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// The GGUF-on-NPU serve route (docs/npu.md, "Open"): `1bit forward-serve -m
+// <model.q4nx> --kernels <dir> -p <port>` serves a repacked Q4NX model through
+// the model-generic forward (npu/forward/q4nx_forward.cpp), behind the same
+// OpenAI endpoints as `1bit unified`.
+#pragma once
+
+namespace onebit {
+
+int run_forward_serve(int argc, char** argv);
+
+}  // namespace onebit