Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions tests/xllm_service/tokenizer/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,13 @@ cc_test(
:tokenizer
GTest::gtest_main
)

cc_test(
NAME
tokenizer_pool_test
SRCS
tokenizer_pool_test.cpp
DEPS
:tokenizer
GTest::gtest_main
)
149 changes: 149 additions & 0 deletions tests/xllm_service/tokenizer/tokenizer_pool_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
/* Copyright 2026 The xLLM Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

https://github.com/xLLM-AI/xllm-service/blob/main/LICENSE

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tokenizer/tokenizer_pool.h"

#include <gtest/gtest.h>

#include <array>
#include <atomic>
#include <memory>
#include <thread>
#include <vector>

namespace xllm_service {
namespace {

struct State {
std::atomic<int32_t> instances{0};
std::atomic<int32_t> overlapping_calls{0};
std::atomic<int32_t> calls_before_warmup{0};
std::array<std::atomic<int32_t>, 4> warmup_flags{};
};

class RecordingTokenizer final : public Tokenizer {
public:
explicit RecordingTokenizer(std::shared_ptr<State> state, int32_t value = 7)
: state_(std::move(state)),
index_(state_->instances.fetch_add(1)),
value_(value) {}

bool encode(const std::string_view& text,
std::vector<int32_t>* ids,
bool add_special_tokens = true) const override {
if (text == "warmup") {
state_->warmup_flags[index_].fetch_or(1);
} else {
if (state_->warmup_flags[index_].load() != 7) {
state_->calls_before_warmup.fetch_add(1);
}
record_call();
}
*ids = {value_, add_special_tokens ? 1 : 0};
return text != "reject";
}

std::string decode(const Slice<int32_t>& ids,
bool skip_special_tokens) const override {
state_->warmup_flags[index_].fetch_or(skip_special_tokens ? 4 : 2);
record_call();
return std::to_string(ids[0]);
}

std::optional<int32_t> token_to_id(const std::string_view&) const override {
return std::nullopt;
}
std::string id_to_token(int32_t) const override { return ""; }
size_t vocab_size() const override { return 10; }
std::unique_ptr<Tokenizer> clone() const override {
return std::make_unique<RecordingTokenizer>(state_, value_);
}

private:
void record_call() const {
if (active_.fetch_add(1) != 0) {
state_->overlapping_calls.fetch_add(1);
}
std::this_thread::yield();
active_.fetch_sub(1);
}

std::shared_ptr<State> state_;
int32_t index_;
int32_t value_;
mutable std::atomic<int32_t> active_{0};
};

TEST(TokenizerPoolTest, InitializesEncodeAndDecodeBeforeServing) {
auto state = std::make_shared<State>();
TokenizerPool pool(std::make_unique<RecordingTokenizer>(state), /*size=*/3);
EXPECT_EQ(state->instances, 3);
for (int32_t index = 0; index < 3; ++index) {
EXPECT_EQ(state->warmup_flags[index], 7);
}
std::vector<int32_t> ids;
EXPECT_TRUE(pool.encode("request", &ids, /*add_special_tokens=*/false));
EXPECT_EQ(ids, (std::vector<int32_t>{7, 0}));
EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/true), "7");
}

TEST(TokenizerPoolTest, ReusesExclusiveInstancesAcrossMoreThreadsThanCapacity) {
auto state = std::make_shared<State>();
TokenizerPool pool(std::make_unique<RecordingTokenizer>(state), /*size=*/2);
std::vector<std::thread> workers;
workers.reserve(8);
for (int32_t index = 0; index < 8; ++index) {
workers.emplace_back([&pool]() {
for (int32_t request = 0; request < 50; ++request) {
std::vector<int32_t> ids;
EXPECT_TRUE(pool.encode("request", &ids));
EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/false), "7");
}
});
}
for (auto& worker : workers) {
worker.join();
}
EXPECT_EQ(state->instances, 2);
EXPECT_EQ(state->overlapping_calls, 0);
EXPECT_EQ(state->calls_before_warmup, 0);
}

TEST(TokenizerPoolTest, ReleasesInstanceWhenEncodeReturnsFalse) {
auto state = std::make_shared<State>();
TokenizerPool pool(std::make_unique<RecordingTokenizer>(state), /*size=*/1);
std::vector<int32_t> ids;
EXPECT_FALSE(pool.encode("reject", &ids));
EXPECT_TRUE(pool.encode("request", &ids));
EXPECT_EQ(pool.decode(ids, /*skip_special_tokens=*/true), "7");
EXPECT_EQ(state->instances, 1);
}

TEST(TokenizerPoolTest, KeepsDifferentModelsIsolatedOnTheSameThread) {
auto first_state = std::make_shared<State>();
auto second_state = std::make_shared<State>();
TokenizerPool first(std::make_unique<RecordingTokenizer>(first_state, 3),
/*size=*/1);
TokenizerPool second(std::make_unique<RecordingTokenizer>(second_state, 9),
/*size=*/1);
std::vector<int32_t> ids;
EXPECT_TRUE(first.encode("request", &ids));
EXPECT_EQ(ids.front(), 3);
EXPECT_TRUE(second.encode("request", &ids));
EXPECT_EQ(ids.front(), 9);
}

} // namespace
} // namespace xllm_service
19 changes: 8 additions & 11 deletions xllm_service/scheduler/scheduler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,11 @@ Scheduler::Scheduler(const Options& options) : options_(options) {
// own tokenization / chat templating. Skip building the local tokenizer and
// chat template so the master does not require a tokenizer.model / template.
if (options_.default_backend_type() != "vllm") {
tokenizer_ = TokenizerFactory::create_tokenizer(options_.tokenizer_path(),
&tokenizer_args_);
CHECK_GT(options_.num_threads(), 0);
tokenizer_pool_ = std::make_unique<TokenizerPool>(
TokenizerFactory::create_tokenizer(options_.tokenizer_path(),
&tokenizer_args_),
static_cast<size_t>(options_.num_threads()));

// Select the chat template by config.json model_type (jinja by default).
model_type = load_model_type(options_.tokenizer_path());
Expand Down Expand Up @@ -171,10 +174,9 @@ bool Scheduler::schedule(std::shared_ptr<Request> request) {
LOG(ERROR) << "Chat template has not configured.";
return false;
}
if (!get_tls_tokenizer()->encode(
request->prompt,
&request->token_ids,
chat_template_->encode_add_special_tokens())) {
if (!tokenizer_pool_->encode(request->prompt,
&request->token_ids,
chat_template_->encode_add_special_tokens())) {
LOG(ERROR) << "Encode prompt failed: " << request->prompt;
return false;
}
Expand Down Expand Up @@ -334,11 +336,6 @@ std::vector<std::string> Scheduler::get_static_prefill_list(
return instance_mgr_->get_static_prefill_list(instance_name);
}

Tokenizer* Scheduler::get_tls_tokenizer() {
thread_local std::unique_ptr<Tokenizer> tls_tokenizer(tokenizer_->clone());
return tls_tokenizer.get();
}

bool Scheduler::record_new_request(std::shared_ptr<ChatCallData> call_data,
std::shared_ptr<Request> request) {
request->latest_generate_time = absl::Now();
Expand Down
6 changes: 2 additions & 4 deletions xllm_service/scheduler/scheduler.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ limitations under the License.
#include "managers/instance_mgr.h"
#include "request/request.h"
#include "response_handler.h"
#include "tokenizer/tokenizer.h"
#include "tokenizer/tokenizer_args.h"
#include "tokenizer/tokenizer_pool.h"

namespace xllm_service {

Expand Down Expand Up @@ -104,8 +104,6 @@ class Scheduler final {
void handle_xservice_watch(const etcd::Response& response,
const uint64_t& prefix_len);

Tokenizer* get_tls_tokenizer();

private:
Options options_;

Expand All @@ -120,7 +118,7 @@ class Scheduler final {

std::shared_ptr<EtcdClient> etcd_client_;

std::unique_ptr<Tokenizer> tokenizer_;
std::unique_ptr<TokenizerPool> tokenizer_pool_;

std::shared_ptr<InstanceMgr> instance_mgr_;

Expand Down
2 changes: 2 additions & 0 deletions xllm_service/tokenizer/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,15 @@ cc_library(
HDRS
tokenizer_args.h
tokenizer.h
tokenizer_pool.h
tokenizer_factory.h
tiktoken_tokenizer.h
sentencepiece_tokenizer.h
fast_tokenizer.h
SRCS
tokenizer_args.cpp
tokenizer_factory.cpp
tokenizer_pool.cpp
tiktoken_tokenizer.cpp
sentencepiece_tokenizer.cpp
fast_tokenizer.cpp
Expand Down
3 changes: 3 additions & 0 deletions xllm_service/tokenizer/fast_tokenizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ bool FastTokenizer::encode(const std::string_view& text,

std::vector<int32_t> ret(result.token_ids, result.token_ids + result.len);
*ids = std::move(ret);
if (result.token_ids != nullptr && result.len > 0) {
tokenizers_free_encode_results(&result, 1);
}

return true;
}
Expand Down
11 changes: 11 additions & 0 deletions xllm_service/tokenizer/tokenizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ limitations under the License.
==============================================================================*/

#pragma once
#include <glog/logging.h>

#include <cstdint>
#include <memory>
#include <optional>
Expand Down Expand Up @@ -43,6 +45,15 @@ class Tokenizer {

virtual size_t vocab_size() const = 0;

// Exercise both directions before this instance is made available to
// requests.
virtual void warmup() const {
std::vector<int32_t> ids;
CHECK(encode("warmup", &ids));
(void)decode(ids, /*skip_special_tokens=*/false);
(void)decode(ids, /*skip_special_tokens=*/true);
}

virtual std::unique_ptr<Tokenizer> clone() const = 0;
};

Expand Down
73 changes: 73 additions & 0 deletions xllm_service/tokenizer/tokenizer_pool.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/* Copyright 2026 The xLLM Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

https://github.com/xLLM-AI/xllm-service/blob/main/LICENSE

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tokenizer/tokenizer_pool.h"

#include <glog/logging.h>

#include <utility>

namespace xllm_service {

class TokenizerPool::Lease final {
public:
Lease(const TokenizerPool& pool, std::unique_ptr<Tokenizer> tokenizer)
: pool_(pool), tokenizer_(std::move(tokenizer)) {}
~Lease() { pool_.available_.push(std::move(tokenizer_)); }

Lease(const Lease&) = delete;
Lease& operator=(const Lease&) = delete;
Lease(Lease&&) = delete;
Lease& operator=(Lease&&) = delete;

const Tokenizer* operator->() const { return tokenizer_.get(); }

private:
const TokenizerPool& pool_;
std::unique_ptr<Tokenizer> tokenizer_;
};

TokenizerPool::TokenizerPool(std::unique_ptr<Tokenizer> tokenizer,
size_t size) {
CHECK(tokenizer != nullptr);
CHECK_GT(size, 0);
for (size_t index = 1; index < size; ++index) {
auto instance = tokenizer->clone();
CHECK(instance != nullptr);
instance->warmup();
available_.push(std::move(instance));
}
tokenizer->warmup();
available_.push(std::move(tokenizer));
}

TokenizerPool::Lease TokenizerPool::acquire() const {
return Lease(*this, available_.pop());
}

bool TokenizerPool::encode(const std::string_view& text,
std::vector<int32_t>* ids,
bool add_special_tokens) const {
auto tokenizer = acquire();
return tokenizer->encode(text, ids, add_special_tokens);
}

std::string TokenizerPool::decode(const Slice<int32_t>& ids,
bool skip_special_tokens) const {
auto tokenizer = acquire();
return tokenizer->decode(ids, skip_special_tokens);
}

} // namespace xllm_service
Loading
Loading