diff --git a/app/serve.cpp b/app/serve.cpp index 5b1c7b4c..de1b1f8b 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -861,15 +861,33 @@ Launch launch_for(const Options& o, const std::string& device, int child_port) { return l; } -// A small llama-server for one RAG role on Vulkan: --embedding for /v1/embeddings, -// --reranking for /v1/rerank. Batch = ubatch: an embedding or rerank input is one pass. +// A small llama-server for one RAG role: --embedding for /v1/embeddings, --reranking for +// /v1/rerank. Batch = ubatch: an embedding or rerank input is one pass. On HRX where the build has +// it (docs/hrx.md), else Vulkan. HRX runs one sequence per batch (docs/hrx.md, "Several sequences +// per batch fail"), so its server takes one slot and 2048-token inputs, the largest prompt chunk +// its matmul kernels take in one pass. +std::string rag_device() { +#if defined(ONEBIT_HRX_SERVER) + return "hrx"; +#else + return "vulkan"; +#endif +} + Launch rag_launch(const Options& o, const std::string& model, const char* role, int port) { Launch l; - l.device = std::string("vulkan ") + role; + const std::string device = rag_device(); + l.device = device + " " + role; l.port = port; - l.argv = {o.llama_server.empty() ? default_llama_server("vulkan") : o.llama_server, "-m", model, "--host", "127.0.0.1", "--port", std::to_string(port), - "--device", "Vulkan0", "-ngl", "99", "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "4", - std::string("--") + role}; + const std::string server = o.llama_server.empty() ? default_llama_server(device) : o.llama_server; + if (device == "hrx") + l.argv = {server, "-m", model, "--host", "127.0.0.1", "--port", std::to_string(port), + "--device", "HRX0", "-ngl", "99", "-c", "2048", "-b", "2048", "-ub", "2048", "-np", "1", + std::string("--") + role}; + else + l.argv = {server, "-m", model, "--host", "127.0.0.1", "--port", std::to_string(port), + "--device", "Vulkan0", "-ngl", "99", "-c", "8192", "-b", "8192", "-ub", "8192", "-np", "4", + std::string("--") + role}; return l; } @@ -1392,8 +1410,8 @@ int serve_child(const Options& given) { srv.Get("/v1/health", health); srv.Get("/v1/models", [&](const httplib::Request&, httplib::Response& res) { json data = json::array({{{"id", id}, {"object", "model"}, {"owned_by", "1bit"}, {"device", device}}}); - if (!o.embed.empty()) data.push_back({{"id", stem(o.embed)}, {"object", "model"}, {"owned_by", "1bit"}, {"device", "vulkan"}, {"role", "embedding"}}); - if (!o.rerank.empty()) data.push_back({{"id", stem(o.rerank)}, {"object", "model"}, {"owned_by", "1bit"}, {"device", "vulkan"}, {"role", "rerank"}}); + if (!o.embed.empty()) data.push_back({{"id", stem(o.embed)}, {"object", "model"}, {"owned_by", "1bit"}, {"device", rag_device()}, {"role", "embedding"}}); + if (!o.rerank.empty()) data.push_back({{"id", stem(o.rerank)}, {"object", "model"}, {"owned_by", "1bit"}, {"device", rag_device()}, {"role", "rerank"}}); res.set_content(json{{"object", "list"}, {"data", data}}.dump(), "application/json"); }); std::vector> children; diff --git a/docs/serve.md b/docs/serve.md index eab7cbb6..ace50f0f 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -372,16 +372,19 @@ rocm); it does not combine with `--laya`, `--adaptive` or `--mmproj`. ## RAG (`--embed`, `--rerank`) `--embed MODEL.gguf` serves `/v1/embeddings` and `--rerank MODEL.gguf` serves `/v1/rerank`, -each from its own llama-server on Vulkan beside the chat model (`--embedding`, -`--reranking`, 4 slots, the whole input in one batch). `/v1/models` lists them with their +each from its own llama-server beside the chat model (`--embedding`, `--reranking`, the whole input +in one batch). In a build with HRX that server runs on `HRX0` with one slot and inputs of up to 2048 +tokens (HRX runs one sequence per batch); a build without HRX keeps Vulkan, 4 slots and 8192 tokens. `/v1/models` lists them with their role. A RAG client embeds its documents, retrieves by cosine similarity, reranks the best few and asks the chat model with them as context, all against the one server: `--embedding` and `--reranking` serve the model itself in that role, as Lemonade loads embedding and reranking models: `1bit serve -m nomic-embed-text-v2-moe.Q8_0.gguf --embedding`. -They run on `vulkan`, `hrx` or `rocm`. On Strix Halo, embeddings work on Vulkan and HRX -(768-dimension vectors). Reranking works on Vulkan, but on HRX jina-reranker-v1-tiny fails: -HRX's JIT can't link the fp32 matmul-with-bias kernel it needs. For now, rerank on Vulkan. +They run on `vulkan`, `hrx` or `rocm`. On Strix Halo (2026-10-02, HRX0 against Vulkan0, two +repeats each): Qwen3-Embedding-0.6B Q8_0 embeddings repeat bit for bit on HRX, cosine 0.99961-0.99988 +to Vulkan's; bge-reranker-v2-m3 Q8_0 scores repeat exactly and rank the same as Vulkan's (8.609 / +-6.756 / -0.401 / -11.020 against 8.614 / -6.757 / -0.361 / -11.019). jina-reranker-v1-tiny still +fails on HRX: its JIT can't link the fp32 matmul-with-bias kernel it needs. ```sh 1bit serve -m Qwen3.8-27B-UD-Q4_K_XL.gguf --mtp mtp-Qwen3.8-27B-Q4_0.gguf \