From b74295fecb5d59a68f4ce666c07759de4cbca0dc Mon Sep 17 00:00:00 2001 From: PixelFlat <329975279+PixelFlat@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:17:57 +0800 Subject: [PATCH] bugfix: initialize service tokenizers before serving requests. --- tests/xllm_service/tokenizer/CMakeLists.txt | 10 ++ .../tokenizer/tokenizer_pool_test.cpp | 149 ++++++++++++++++++ xllm_service/scheduler/scheduler.cpp | 19 +-- xllm_service/scheduler/scheduler.h | 6 +- xllm_service/tokenizer/CMakeLists.txt | 2 + xllm_service/tokenizer/fast_tokenizer.cpp | 3 + xllm_service/tokenizer/tokenizer.h | 11 ++ xllm_service/tokenizer/tokenizer_pool.cpp | 73 +++++++++ xllm_service/tokenizer/tokenizer_pool.h | 47 ++++++ .../tokenizer/tokenizers/tokenizers.h | 3 + 10 files changed, 308 insertions(+), 15 deletions(-) create mode 100644 tests/xllm_service/tokenizer/tokenizer_pool_test.cpp create mode 100644 xllm_service/tokenizer/tokenizer_pool.cpp create mode 100644 xllm_service/tokenizer/tokenizer_pool.h diff --git a/tests/xllm_service/tokenizer/CMakeLists.txt b/tests/xllm_service/tokenizer/CMakeLists.txt index 379e0366..b9c472f4 100644 --- a/tests/xllm_service/tokenizer/CMakeLists.txt +++ b/tests/xllm_service/tokenizer/CMakeLists.txt @@ -9,3 +9,13 @@ cc_test( :tokenizer GTest::gtest_main ) + +cc_test( + NAME + tokenizer_pool_test + SRCS + tokenizer_pool_test.cpp + DEPS + :tokenizer + GTest::gtest_main +) diff --git a/tests/xllm_service/tokenizer/tokenizer_pool_test.cpp b/tests/xllm_service/tokenizer/tokenizer_pool_test.cpp new file mode 100644 index 00000000..e191050a --- /dev/null +++ b/tests/xllm_service/tokenizer/tokenizer_pool_test.cpp @@ -0,0 +1,149 @@ +/* Copyright 2026 The xLLM Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/xLLM-AI/xllm-service/blob/main/LICENSE + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tokenizer/tokenizer_pool.h" + +#include + +#include +#include +#include +#include +#include + +namespace xllm_service { +namespace { + +struct State { + std::atomic instances{0}; + std::atomic overlapping_calls{0}; + std::atomic calls_before_warmup{0}; + std::array, 4> warmup_flags{}; +}; + +class RecordingTokenizer final : public Tokenizer { + public: + explicit RecordingTokenizer(std::shared_ptr state, int32_t value = 7) + : state_(std::move(state)), + index_(state_->instances.fetch_add(1)), + value_(value) {} + + bool encode(const std::string_view& text, + std::vector* ids, + bool add_special_tokens = true) const override { + if (text == "warmup") { + state_->warmup_flags[index_].fetch_or(1); + } else { + if (state_->warmup_flags[index_].load() != 7) { + state_->calls_before_warmup.fetch_add(1); + } + record_call(); + } + *ids = {value_, add_special_tokens ? 1 : 0}; + return text != "reject"; + } + + std::string decode(const Slice& ids, + bool skip_special_tokens) const override { + state_->warmup_flags[index_].fetch_or(skip_special_tokens ? 4 : 2); + record_call(); + return std::to_string(ids[0]); + } + + std::optional token_to_id(const std::string_view&) const override { + return std::nullopt; + } + std::string id_to_token(int32_t) const override { return ""; } + size_t vocab_size() const override { return 10; } + std::unique_ptr clone() const override { + return std::make_unique(state_, value_); + } + + private: + void record_call() const { + if (active_.fetch_add(1) != 0) { + state_->overlapping_calls.fetch_add(1); + } + std::this_thread::yield(); + active_.fetch_sub(1); + } + + std::shared_ptr state_; + int32_t index_; + int32_t value_; + mutable std::atomic active_{0}; +}; + +TEST(TokenizerPoolTest, InitializesEncodeAndDecodeBeforeServing) { + auto state = std::make_shared(); + TokenizerPool pool(std::make_unique(state), /*size=*/3); + EXPECT_EQ(state->instances, 3); + for (int32_t index = 0; index < 3; ++index) { + EXPECT_EQ(state->warmup_flags[index], 7); + } + std::vector ids; + EXPECT_TRUE(pool.encode("request", &ids, /*add_special_tokens=*/false)); + EXPECT_EQ(ids, (std::vector{7, 0})); + EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/true), "7"); +} + +TEST(TokenizerPoolTest, ReusesExclusiveInstancesAcrossMoreThreadsThanCapacity) { + auto state = std::make_shared(); + TokenizerPool pool(std::make_unique(state), /*size=*/2); + std::vector workers; + workers.reserve(8); + for (int32_t index = 0; index < 8; ++index) { + workers.emplace_back([&pool]() { + for (int32_t request = 0; request < 50; ++request) { + std::vector ids; + EXPECT_TRUE(pool.encode("request", &ids)); + EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/false), "7"); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + EXPECT_EQ(state->instances, 2); + EXPECT_EQ(state->overlapping_calls, 0); + EXPECT_EQ(state->calls_before_warmup, 0); +} + +TEST(TokenizerPoolTest, ReleasesInstanceWhenEncodeReturnsFalse) { + auto state = std::make_shared(); + TokenizerPool pool(std::make_unique(state), /*size=*/1); + std::vector ids; + EXPECT_FALSE(pool.encode("reject", &ids)); + EXPECT_TRUE(pool.encode("request", &ids)); + EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/true), "7"); + EXPECT_EQ(state->instances, 1); +} + +TEST(TokenizerPoolTest, KeepsDifferentModelsIsolatedOnTheSameThread) { + auto first_state = std::make_shared(); + auto second_state = std::make_shared(); + TokenizerPool first(std::make_unique(first_state, 3), + /*size=*/1); + TokenizerPool second(std::make_unique(second_state, 9), + /*size=*/1); + std::vector ids; + EXPECT_TRUE(first.encode("request", &ids)); + EXPECT_EQ(ids.front(), 3); + EXPECT_TRUE(second.encode("request", &ids)); + EXPECT_EQ(ids.front(), 9); +} + +} // namespace +} // namespace xllm_service diff --git a/xllm_service/scheduler/scheduler.cpp b/xllm_service/scheduler/scheduler.cpp index 0b011269..a8b59ef1 100644 --- a/xllm_service/scheduler/scheduler.cpp +++ b/xllm_service/scheduler/scheduler.cpp @@ -48,8 +48,11 @@ Scheduler::Scheduler(const Options& options) : options_(options) { // own tokenization / chat templating. Skip building the local tokenizer and // chat template so the master does not require a tokenizer.model / template. if (options_.default_backend_type() != "vllm") { - tokenizer_ = TokenizerFactory::create_tokenizer(options_.tokenizer_path(), - &tokenizer_args_); + CHECK_GT(options_.num_threads(), 0); + tokenizer_pool_ = std::make_unique( + TokenizerFactory::create_tokenizer(options_.tokenizer_path(), + &tokenizer_args_), + static_cast(options_.num_threads())); // Select the chat template by config.json model_type (jinja by default). model_type = load_model_type(options_.tokenizer_path()); @@ -171,10 +174,9 @@ bool Scheduler::schedule(std::shared_ptr request) { LOG(ERROR) << "Chat template has not configured."; return false; } - if (!get_tls_tokenizer()->encode( - request->prompt, - &request->token_ids, - chat_template_->encode_add_special_tokens())) { + if (!tokenizer_pool_->encode(request->prompt, + &request->token_ids, + chat_template_->encode_add_special_tokens())) { LOG(ERROR) << "Encode prompt failed: " << request->prompt; return false; } @@ -334,11 +336,6 @@ std::vector Scheduler::get_static_prefill_list( return instance_mgr_->get_static_prefill_list(instance_name); } -Tokenizer* Scheduler::get_tls_tokenizer() { - thread_local std::unique_ptr tls_tokenizer(tokenizer_->clone()); - return tls_tokenizer.get(); -} - bool Scheduler::record_new_request(std::shared_ptr call_data, std::shared_ptr request) { request->latest_generate_time = absl::Now(); diff --git a/xllm_service/scheduler/scheduler.h b/xllm_service/scheduler/scheduler.h index 970bc7a6..8658b398 100644 --- a/xllm_service/scheduler/scheduler.h +++ b/xllm_service/scheduler/scheduler.h @@ -27,8 +27,8 @@ limitations under the License. #include "managers/instance_mgr.h" #include "request/request.h" #include "response_handler.h" -#include "tokenizer/tokenizer.h" #include "tokenizer/tokenizer_args.h" +#include "tokenizer/tokenizer_pool.h" namespace xllm_service { @@ -104,8 +104,6 @@ class Scheduler final { void handle_xservice_watch(const etcd::Response& response, const uint64_t& prefix_len); - Tokenizer* get_tls_tokenizer(); - private: Options options_; @@ -120,7 +118,7 @@ class Scheduler final { std::shared_ptr etcd_client_; - std::unique_ptr tokenizer_; + std::unique_ptr tokenizer_pool_; std::shared_ptr instance_mgr_; diff --git a/xllm_service/tokenizer/CMakeLists.txt b/xllm_service/tokenizer/CMakeLists.txt index 52070a8a..23715b57 100644 --- a/xllm_service/tokenizer/CMakeLists.txt +++ b/xllm_service/tokenizer/CMakeLists.txt @@ -8,6 +8,7 @@ cc_library( HDRS tokenizer_args.h tokenizer.h + tokenizer_pool.h tokenizer_factory.h tiktoken_tokenizer.h sentencepiece_tokenizer.h @@ -15,6 +16,7 @@ cc_library( SRCS tokenizer_args.cpp tokenizer_factory.cpp + tokenizer_pool.cpp tiktoken_tokenizer.cpp sentencepiece_tokenizer.cpp fast_tokenizer.cpp diff --git a/xllm_service/tokenizer/fast_tokenizer.cpp b/xllm_service/tokenizer/fast_tokenizer.cpp index ecc79cbd..bd5add7c 100644 --- a/xllm_service/tokenizer/fast_tokenizer.cpp +++ b/xllm_service/tokenizer/fast_tokenizer.cpp @@ -41,6 +41,9 @@ bool FastTokenizer::encode(const std::string_view& text, std::vector ret(result.token_ids, result.token_ids + result.len); *ids = std::move(ret); + if (result.token_ids != nullptr && result.len > 0) { + tokenizers_free_encode_results(&result, 1); + } return true; } diff --git a/xllm_service/tokenizer/tokenizer.h b/xllm_service/tokenizer/tokenizer.h index 806d5ccc..6863d487 100644 --- a/xllm_service/tokenizer/tokenizer.h +++ b/xllm_service/tokenizer/tokenizer.h @@ -15,6 +15,8 @@ limitations under the License. ==============================================================================*/ #pragma once +#include + #include #include #include @@ -43,6 +45,15 @@ class Tokenizer { virtual size_t vocab_size() const = 0; + // Exercise both directions before this instance is made available to + // requests. + virtual void warmup() const { + std::vector ids; + CHECK(encode("warmup", &ids)); + (void)decode(ids, /*skip_special_tokens=*/false); + (void)decode(ids, /*skip_special_tokens=*/true); + } + virtual std::unique_ptr clone() const = 0; }; diff --git a/xllm_service/tokenizer/tokenizer_pool.cpp b/xllm_service/tokenizer/tokenizer_pool.cpp new file mode 100644 index 00000000..d2342298 --- /dev/null +++ b/xllm_service/tokenizer/tokenizer_pool.cpp @@ -0,0 +1,73 @@ +/* Copyright 2026 The xLLM Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/xLLM-AI/xllm-service/blob/main/LICENSE + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tokenizer/tokenizer_pool.h" + +#include + +#include + +namespace xllm_service { + +class TokenizerPool::Lease final { + public: + Lease(const TokenizerPool& pool, std::unique_ptr tokenizer) + : pool_(pool), tokenizer_(std::move(tokenizer)) {} + ~Lease() { pool_.available_.push(std::move(tokenizer_)); } + + Lease(const Lease&) = delete; + Lease& operator=(const Lease&) = delete; + Lease(Lease&&) = delete; + Lease& operator=(Lease&&) = delete; + + const Tokenizer* operator->() const { return tokenizer_.get(); } + + private: + const TokenizerPool& pool_; + std::unique_ptr tokenizer_; +}; + +TokenizerPool::TokenizerPool(std::unique_ptr tokenizer, + size_t size) { + CHECK(tokenizer != nullptr); + CHECK_GT(size, 0); + for (size_t index = 1; index < size; ++index) { + auto instance = tokenizer->clone(); + CHECK(instance != nullptr); + instance->warmup(); + available_.push(std::move(instance)); + } + tokenizer->warmup(); + available_.push(std::move(tokenizer)); +} + +TokenizerPool::Lease TokenizerPool::acquire() const { + return Lease(*this, available_.pop()); +} + +bool TokenizerPool::encode(const std::string_view& text, + std::vector* ids, + bool add_special_tokens) const { + auto tokenizer = acquire(); + return tokenizer->encode(text, ids, add_special_tokens); +} + +std::string TokenizerPool::decode(const Slice& ids, + bool skip_special_tokens) const { + auto tokenizer = acquire(); + return tokenizer->decode(ids, skip_special_tokens); +} + +} // namespace xllm_service diff --git a/xllm_service/tokenizer/tokenizer_pool.h b/xllm_service/tokenizer/tokenizer_pool.h new file mode 100644 index 00000000..99b061f6 --- /dev/null +++ b/xllm_service/tokenizer/tokenizer_pool.h @@ -0,0 +1,47 @@ +/* Copyright 2026 The xLLM Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://github.com/xLLM-AI/xllm-service/blob/main/LICENSE + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include "common/concurrent_queue.h" +#include "tokenizer/tokenizer.h" + +namespace xllm_service { + +// Instances are initialized eagerly and borrowed exclusively for one operation. +// Exhaustion waits for an existing instance instead of cloning on a request. +class TokenizerPool final { + public: + TokenizerPool(std::unique_ptr tokenizer, size_t size); + + bool encode(const std::string_view& text, + std::vector* ids, + bool add_special_tokens = true) const; + std::string decode(const Slice& ids, bool skip_special_tokens) const; + + private: + class Lease; + Lease acquire() const; + + mutable ConcurrentQueue> available_; +}; + +} // namespace xllm_service diff --git a/xllm_service/tokenizer/tokenizers/tokenizers.h b/xllm_service/tokenizer/tokenizers/tokenizers.h index d9dedc16..9dd59b66 100644 --- a/xllm_service/tokenizer/tokenizers/tokenizers.h +++ b/xllm_service/tokenizer/tokenizers/tokenizers.h @@ -41,6 +41,9 @@ void tokenizers_encode(TokenizerHandle handle, int add_special_token, TokenizerEncodeResult* result); +void tokenizers_free_encode_results(TokenizerEncodeResult* results, + size_t num_seqs); + void tokenizers_decode(TokenizerHandle handle, const uint32_t* data, size_t len,