From 58014b61c217fe377e35f713a41ae3cf7a11f9af Mon Sep 17 00:00:00 2001 From: yutingzeng Date: Tue, 8 Sep 2026 19:41:39 +0800 Subject: [PATCH] bugfix: normalize undefined template checks. Rewrite Jinja undefined tests to Minja-supported none tests before rendering. Cover missing and false enable_thinking values. --- .../jinja_chat_template_test.cpp | 497 ++++++++++++++++++ .../chat_template/jinja_chat_template.cpp | 481 ++++++++++++++++- 2 files changed, 976 insertions(+), 2 deletions(-) diff --git a/tests/xllm_service/chat_template/jinja_chat_template_test.cpp b/tests/xllm_service/chat_template/jinja_chat_template_test.cpp index fa0c81b..513ac84 100644 --- a/tests/xllm_service/chat_template/jinja_chat_template_test.cpp +++ b/tests/xllm_service/chat_template/jinja_chat_template_test.cpp @@ -88,6 +88,503 @@ TEST(JinjaChatTemplate, ApplyChatTemplateKwargs) { EXPECT_EQ(result.value(), "hello"); } +TEST(JinjaChatTemplate, SupportsUndefinedTestForOptionalKwargs) { + const std::string template_str = + "{% if enable_thinking is undefined or enable_thinking is true %}" + "thinking" + "{% else %}no_thinking{% endif %}" + "{{ messages[0]['content'] }}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = {{{"role", "user"}, {"content", "hello"}}}; + const nlohmann::ordered_json tools = nlohmann::ordered_json::array(); + + const nlohmann::ordered_json empty_kwargs = nlohmann::json::object(); + auto missing_value_result = template_.apply(messages, tools, empty_kwargs); + ASSERT_TRUE(missing_value_result.has_value()); + EXPECT_EQ(missing_value_result.value(), "thinkinghello"); + + const nlohmann::ordered_json false_kwargs = {{"enable_thinking", false}}; + auto false_value_result = template_.apply(messages, tools, false_kwargs); + ASSERT_TRUE(false_value_result.has_value()); + EXPECT_EQ(false_value_result.value(), "no_thinkinghello"); +} + +TEST(JinjaChatTemplate, SupportsWhitespaceInUndefinedTests) { + const std::string template_str = + "{% if enable_thinking is\nundefined %}undefined" + "{% elif enable_thinking is not\tundefined %}defined" + "{% else %}unexpected{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + + const nlohmann::ordered_json empty_kwargs = nlohmann::json::object(); + auto missing_value_result = template_.apply(messages, tools, empty_kwargs); + ASSERT_TRUE(missing_value_result.has_value()); + EXPECT_EQ(missing_value_result.value(), "undefined"); + + const nlohmann::ordered_json false_kwargs = {{"enable_thinking", false}}; + auto false_value_result = template_.apply(messages, tools, false_kwargs); + ASSERT_TRUE(false_value_result.has_value()); + EXPECT_EQ(false_value_result.value(), "defined"); +} + +TEST(JinjaChatTemplate, SupportsUndefinedTestAfterDelimiterInStringLiteral) { + const std::string template_str = + "{% if 'value%}' == 'value%}' and enable_thinking is undefined %}" + "undefined{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "undefined"); +} + +TEST(JinjaChatTemplate, SupportsPunctuationBeforeUndefinedTest) { + const std::string template_str = + "{% if (enable_thinking)is undefined %}missing" + "{% else %}unexpected{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + + const nlohmann::ordered_json empty_kwargs = nlohmann::json::object(); + auto missing_result = template_.apply(messages, tools, empty_kwargs); + ASSERT_TRUE(missing_result.has_value()); + EXPECT_EQ(missing_result.value(), "missing"); +} + +TEST(JinjaChatTemplate, PreservesNullInUndefinedTests) { + const std::string template_str = + "{% if enable_thinking is undefined %}missing" + "{% elif enable_thinking is not undefined %}defined" + "{% else %}unexpected{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json null_kwargs = {{"enable_thinking", nullptr}}; + auto null_result = template_.apply(messages, tools, null_kwargs); + ASSERT_TRUE(null_result.has_value()); + EXPECT_EQ(null_result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesUnaryNotPrecedenceInUndefinedTests) { + const std::string template_str = + "{% if not enable_thinking is undefined %}defined" + "{% else %}missing{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + + const nlohmann::ordered_json empty_kwargs = nlohmann::json::object(); + auto missing_result = template_.apply(messages, tools, empty_kwargs); + ASSERT_TRUE(missing_result.has_value()); + EXPECT_EQ(missing_result.value(), "missing"); + + const nlohmann::ordered_json null_kwargs = {{"enable_thinking", nullptr}}; + auto null_result = template_.apply(messages, tools, null_kwargs); + ASSERT_TRUE(null_result.has_value()); + EXPECT_EQ(null_result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesNestedNullInUndefinedTests) { + const std::string template_str = + "{% if options.enable_thinking is undefined %}missing" + "{% elif options.enable_thinking is not undefined %}defined" + "{% else %}unexpected{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json null_kwargs = { + {"options", {{"enable_thinking", nullptr}}}}; + auto null_result = template_.apply(messages, tools, null_kwargs); + ASSERT_TRUE(null_result.has_value()); + EXPECT_EQ(null_result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesLocalNullInUndefinedTests) { + const std::string template_str = + "{% set value = none %}" + "{% if value is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesWhitespaceControlledLocalNull) { + const std::string template_str = + "{% set value = none -%}" + "{% if value is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesWhitespaceControlAfterLocalNullInjection) { + const std::string template_str = + "{% set value = none -%}\n" + "{% if value is undefined %}missing{% else %}Hello{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "Hello"); +} + +TEST(JinjaChatTemplate, DoesNotTreatNoneLiteralAsUndefined) { + const std::string template_str = + "{% if none is undefined %}unexpected" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, DoesNotTreatUppercaseNoneLiteralAsUndefined) { + const std::string template_str = + "{% if None is undefined %}unexpected" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, SupportsUndefinedTestCallSyntax) { + const std::string template_str = + "{% if enable_thinking is undefined() %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "missing"); +} + +TEST(JinjaChatTemplate, RemovesPresenceAfterLocalBecomesUndefined) { + const std::string template_str = + "{% set value = none %}" + "{% set value = missing %}" + "{% if value is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "missing"); +} + +TEST(JinjaChatTemplate, PreservesNullAssignedFromContextPath) { + const std::string template_str = + "{% set value = source %}" + "{% if value is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = {{"source", nullptr}}; + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, RemovesDescendantPresenceAfterLocalOverwrite) { + const std::string template_str = + "{% set options = {} %}" + "{% if options.flag is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = {{"options", {{"flag", nullptr}}}}; + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "missing"); +} + +TEST(JinjaChatTemplate, PropagatesDescendantPresenceThroughLocalAlias) { + const std::string template_str = + "{% set alias = options %}" + "{% if alias.flag is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = {{"options", {{"flag", nullptr}}}}; + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, PreservesUnderscoreBindingDuringPresenceTracking) { + const std::string template_str = + "{% set _ = 'visible' %}" + "{% set value = none %}" + "{{ _ }}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "visible"); +} + +TEST(JinjaChatTemplate, SupportsNestedBracesBeforeUndefinedTest) { + const std::string template_str = "{{ {\"a\": {}} or opt is undefined }}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "{'a': {}}"); +} + +TEST(JinjaChatTemplate, PreservesBinaryOperandInUndefinedTest) { + const std::string template_str = + "{% if 1 + value is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = {{"value", 2}}; + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "defined"); +} + +TEST(JinjaChatTemplate, DoesNotConflateLiteralDotsInPresencePaths) { + const std::string template_str = + "{% if options.enable_thinking is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = {{"options", nlohmann::json::object()}, + {"options.enable_thinking", false}}; + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "missing"); +} + +TEST(JinjaChatTemplate, DoesNotTreatFilterNameAsContextPath) { + const std::string template_str = + "{% if value | default is undefined %}missing" + "{% else %}defined{% endif %}"; + + TokenizerArgs args; + args.chat_template(template_str); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = nlohmann::json::array(); + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = nlohmann::json::object(); + + auto result = template_.apply(messages, tools, kwargs); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), "missing"); +} + +TEST(JinjaChatTemplate, RejectsNonObjectChatTemplateKwargs) { + TokenizerArgs args; + args.chat_template("{{ messages[0]['content'] }}"); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = {{{"role", "user"}, {"content", "hello"}}}; + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json invalid_kwargs = nlohmann::json::array(); + + auto result = template_.apply(messages, tools, invalid_kwargs); + EXPECT_FALSE(result.has_value()); +} + +TEST(JinjaChatTemplate, RejectsReservedChatTemplateKwargsKey) { + TokenizerArgs args; + args.chat_template("{{ messages[0]['content'] }}"); + args.bos_token(""); + args.eos_token(""); + JinjaChatTemplate template_(args); + + nlohmann::ordered_json messages = {{{"role", "user"}, {"content", "hello"}}}; + const nlohmann::ordered_json tools = nlohmann::json::array(); + const nlohmann::ordered_json kwargs = { + {"__xllm_chat_template_kwargs_presence", "user_value"}}; + + auto result = template_.apply(messages, tools, kwargs); + EXPECT_FALSE(result.has_value()); +} + TEST(JinjaChatTemplate, MakesThinkingAndEffortAvailableInExtraContext) { TokenizerArgs args; args.chat_template( diff --git a/xllm_service/chat_template/jinja_chat_template.cpp b/xllm_service/chat_template/jinja_chat_template.cpp index 8809c8d..6a2a4cd 100644 --- a/xllm_service/chat_template/jinja_chat_template.cpp +++ b/xllm_service/chat_template/jinja_chat_template.cpp @@ -18,13 +18,72 @@ limitations under the License. #include #include +#include +#include #include #include +#include #include namespace xllm_service { namespace { +constexpr std::string_view kChatTemplateKwargsPresence = + "__xllm_chat_template_kwargs_presence"; +constexpr std::string_view kChatTemplateKwargsPresenceState = + "__xllm_chat_template_kwargs_presence_state"; + +std::string_view trim_whitespace(std::string_view text); + +std::string escape_json_pointer_token(std::string_view token) { + std::string escaped; + for (const char character : token) { + if (character == '~') { + escaped += "~0"; + } else if (character == '/') { + escaped += "~1"; + } else { + escaped += character; + } + } + return escaped; +} + +std::string presence_path_for_context_path(std::string_view context_path) { + std::string path; + size_t component_start = 0; + while (component_start < context_path.size()) { + const size_t component_end = context_path.find('.', component_start); + const std::string_view component = + context_path.substr(component_start, component_end - component_start); + path += "/" + escape_json_pointer_token(component); + if (component_end == std::string_view::npos) { + break; + } + component_start = component_end + 1; + } + return path; +} + +void append_present_paths(const nlohmann::ordered_json& value, + const std::string& path, + nlohmann::ordered_json& paths) { + if (value.is_object()) { + for (const auto& item : value.items()) { + const std::string item_path = + path + "/" + escape_json_pointer_token(item.key()); + paths.push_back(item_path); + append_present_paths(item.value(), item_path, paths); + } + } else if (value.is_array()) { + for (size_t index = 0; index < value.size(); ++index) { + const std::string item_path = path + "/" + std::to_string(index); + paths.push_back(item_path); + append_present_paths(value[index], item_path, paths); + } + } +} + bool get_str_field(const nlohmann::ordered_json& json, const char* key, std::string* value) { @@ -128,12 +187,413 @@ nlohmann::ordered_json with_thinking_fallback( return fallback; } +void replace_undefined_tests(std::string& block) { + constexpr std::string_view kIs = "is"; + constexpr std::string_view kNot = "not"; + constexpr std::string_view kUndefined = "undefined"; + constexpr std::string_view kIsNone = "is none"; + constexpr std::string_view kIsNotNone = "is not none"; + + const auto is_whitespace = [](char character) { + return std::isspace(static_cast(character)) != 0; + }; + const auto is_identifier_character = [](char character) { + return std::isalnum(static_cast(character)) != 0 || + character == '_'; + }; + const auto is_context_path_character = + [&is_identifier_character](char character) { + return is_identifier_character(character) || character == '.'; + }; + const auto skip_whitespace = [&is_whitespace](const std::string& text, + size_t pos) { + while (pos < text.size() && is_whitespace(text[pos])) { + ++pos; + } + return pos; + }; + const auto context_path_before = [&is_whitespace, &is_context_path_character]( + const std::string& text, size_t pos) { + while (pos > 0 && is_whitespace(text[pos - 1])) { + --pos; + } + const size_t path_end = pos; + while (pos > 0 && is_context_path_character(text[pos - 1])) { + --pos; + } + if (pos == path_end || text[pos] == '.' || text[path_end - 1] == '.') { + return std::string_view(); + } + size_t preceding_pos = pos; + while (preceding_pos > 0 && is_whitespace(text[preceding_pos - 1])) { + --preceding_pos; + } + if (preceding_pos > 0) { + const char preceding = text[preceding_pos - 1]; + if (preceding == '|' || preceding == '+' || preceding == '-' || + preceding == '*' || preceding == '/' || preceding == '%' || + preceding == '~') { + return std::string_view(); + } + } + return std::string_view(text.data() + pos, path_end - pos); + }; + + char quote = '\0'; + for (size_t pos = 0; pos < block.size();) { + const char current = block[pos]; + if (quote != '\0') { + if (current == '\\') { + pos += 2; + } else if (current == quote) { + quote = '\0'; + ++pos; + } else { + ++pos; + } + continue; + } + if (current == '\'' || current == '"') { + quote = current; + ++pos; + continue; + } + + if (block.compare(pos, kIs.size(), kIs) != 0 || + (pos > 0 && is_identifier_character(block[pos - 1]))) { + ++pos; + continue; + } + + size_t token_end = pos + kIs.size(); + const size_t test_pos = skip_whitespace(block, token_end); + if (test_pos == token_end) { + ++pos; + continue; + } + + bool has_not = false; + if (block.compare(test_pos, kNot.size(), kNot) == 0) { + token_end = test_pos + kNot.size(); + const size_t undefined_pos = skip_whitespace(block, token_end); + if (undefined_pos == token_end) { + ++pos; + continue; + } + token_end = undefined_pos; + has_not = true; + } else { + token_end = test_pos; + } + + if (block.compare(token_end, kUndefined.size(), kUndefined) != 0 || + (token_end + kUndefined.size() < block.size() && + is_identifier_character(block[token_end + kUndefined.size()]))) { + ++pos; + continue; + } + + const std::string_view context_path = context_path_before(block, pos); + size_t replace_end = token_end + kUndefined.size(); + const size_t call_pos = skip_whitespace(block, replace_end); + if (call_pos < block.size() && block[call_pos] == '(') { + const size_t close_paren_pos = skip_whitespace(block, call_pos + 1); + if (close_paren_pos < block.size() && block[close_paren_pos] == ')') { + replace_end = close_paren_pos + 1; + } + } + if (context_path.empty()) { + const std::string_view replacement = has_not ? kIsNotNone : kIsNone; + block.replace(pos, replace_end - pos, replacement); + pos += replacement.size(); + continue; + } + + const size_t context_path_pos = + static_cast(context_path.data() - block.data()); + if (context_path == "none" || context_path == "None" || + context_path == "true" || context_path == "True" || + context_path == "false" || context_path == "False") { + const std::string_view replacement = has_not ? "true" : "false"; + block.replace( + context_path_pos, replace_end - context_path_pos, replacement); + pos = context_path_pos + replacement.size(); + continue; + } + std::string replacement = "(" + std::string(context_path) + " "; + replacement += has_not ? kIsNotNone : kIsNone; + replacement += has_not ? " or '" : " and '"; + replacement += presence_path_for_context_path(context_path); + replacement += has_not ? "' in " : "' not in "; + replacement += kChatTemplateKwargsPresence; + replacement += ")"; + block.replace( + context_path_pos, replace_end - context_path_pos, replacement); + pos = context_path_pos + replacement.size(); + } +} + +struct LocalAssignment { + std::string name; + std::string value_expression; +}; + +std::optional local_assignment(const std::string& block) { + std::string_view content(block.data() + 2, block.size() - 4); + content = trim_whitespace(content); + if (!content.empty() && content.front() == '-') { + content.remove_prefix(1); + content = trim_whitespace(content); + } + constexpr std::string_view kSet = "set"; + if (content.substr(0, kSet.size()) != kSet || + (content.size() > kSet.size() && + std::isspace(static_cast(content[kSet.size()])) == 0)) { + return std::nullopt; + } + content.remove_prefix(kSet.size()); + content = trim_whitespace(content); + size_t name_end = 0; + while (name_end < content.size() && + (std::isalnum(static_cast(content[name_end])) != 0 || + content[name_end] == '_')) { + ++name_end; + } + if (name_end == 0) { + return std::nullopt; + } + const std::string name(content.substr(0, name_end)); + content.remove_prefix(name_end); + content = trim_whitespace(content); + if (content.empty() || content.front() != '=') { + return std::nullopt; + } + content.remove_prefix(1); + content = trim_whitespace(content); + if (!content.empty() && content.back() == '-') { + content.remove_suffix(1); + content = trim_whitespace(content); + } + return LocalAssignment{std::move(name), std::string(content)}; +} + +bool is_context_path(std::string_view expression) { + if (expression.empty() || expression.front() == '.' || + expression.back() == '.') { + return false; + } + for (const char character : expression) { + if (std::isalnum(static_cast(character)) == 0 && + character != '_' && character != '.') { + return false; + } + } + return true; +} + +bool is_defined_literal(std::string_view expression) { + while (expression.size() >= 2 && expression.front() == '(' && + expression.back() == ')') { + expression.remove_prefix(1); + expression.remove_suffix(1); + expression = trim_whitespace(expression); + } + return expression == "none" || expression == "None" || expression == "true" || + expression == "True" || expression == "false" || expression == "False"; +} + +size_t find_block_end(const std::string& chat_template, + size_t content_pos, + std::string_view close_delimiter) { + char quote = '\0'; + const bool is_output_block = close_delimiter == "}}"; + int32_t brace_depth = 0; + for (size_t pos = content_pos; pos < chat_template.size(); ++pos) { + const char current = chat_template[pos]; + if (quote != '\0') { + if (current == '\\') { + ++pos; + } else if (current == quote) { + quote = '\0'; + } + continue; + } + if (current == '\'' || current == '"') { + quote = current; + continue; + } + if (chat_template.compare(pos, close_delimiter.size(), close_delimiter) == + 0 && + (!is_output_block || brace_depth == 0)) { + return pos; + } + if (is_output_block && current == '{') { + ++brace_depth; + } else if (is_output_block && current == '}' && brace_depth > 0) { + --brace_depth; + } + } + return std::string::npos; +} + +std::string_view trim_whitespace(std::string_view text) { + while (!text.empty() && + std::isspace(static_cast(text.front())) != 0) { + text.remove_prefix(1); + } + while (!text.empty() && + std::isspace(static_cast(text.back())) != 0) { + text.remove_suffix(1); + } + return text; +} + +bool is_statement(const std::string& block, std::string_view statement) { + std::string_view content(block.data() + 2, block.size() - 4); + content = trim_whitespace(content); + if (!content.empty() && content.front() == '-') { + content.remove_prefix(1); + content = trim_whitespace(content); + } + if (content.size() < statement.size() || + content.substr(0, statement.size()) != statement) { + return false; + } + if (content.size() == statement.size()) { + return true; + } + const char next = content[statement.size()]; + return std::isspace(static_cast(next)) != 0 || next == '-'; +} + +size_t find_raw_end(const std::string& chat_template, size_t search_pos) { + while (search_pos < chat_template.size()) { + const size_t block_pos = chat_template.find("{%", search_pos); + if (block_pos == std::string::npos) { + return std::string::npos; + } + const size_t close_pos = chat_template.find("%}", block_pos + 2); + if (close_pos == std::string::npos) { + return std::string::npos; + } + const std::string block = chat_template.substr( + block_pos, close_pos + std::string_view("%}").size() - block_pos); + if (is_statement(block, "endraw")) { + return close_pos + std::string_view("%}").size(); + } + search_pos = block_pos + 2; + } + return std::string::npos; +} + +std::string normalize_minja_tests(std::string chat_template) { + size_t search_pos = 0; + while (search_pos < chat_template.size()) { + const size_t block_pos = chat_template.find('{', search_pos); + if (block_pos == std::string::npos) { + break; + } + + if (block_pos + 1 >= chat_template.size()) { + break; + } + + const char block_type = chat_template[block_pos + 1]; + if (block_type == '#') { + const size_t close_pos = chat_template.find("#}", block_pos + 2); + if (close_pos == std::string::npos) { + break; + } + search_pos = close_pos + std::string_view("#}").size(); + continue; + } + if (block_type != '{' && block_type != '%') { + search_pos = block_pos + 1; + continue; + } + + const std::string_view close = block_type == '{' ? "}}" : "%}"; + const size_t close_pos = + find_block_end(chat_template, block_pos + 2, close); + if (close_pos == std::string::npos) { + break; + } + + std::string block = + chat_template.substr(block_pos, close_pos + close.size() - block_pos); + if (block_type == '%' && is_statement(block, "raw")) { + const size_t raw_end_pos = find_raw_end(chat_template, close_pos + 2); + if (raw_end_pos == std::string::npos) { + break; + } + search_pos = raw_end_pos; + continue; + } + + // Qwen templates use Jinja's `is undefined` test for optional arguments. + // Minja represents missing arguments as null and supports `is none`. + replace_undefined_tests(block); + if (block_type == '%') { + const auto assignment = local_assignment(block); + if (assignment.has_value()) { + const bool trim_trailing_whitespace = + block.size() >= 3 && block.compare(block.size() - 3, 3, "-%}") == 0; + if (trim_trailing_whitespace) { + block.erase(block.size() - 3, 1); + } + const std::string presence_path = + presence_path_for_context_path(assignment->name); + const std::string presence_name(kChatTemplateKwargsPresence); + const std::string state_name(kChatTemplateKwargsPresenceState); + block += "{% set " + state_name + + " = namespace(paths=[]) %}" + "{% for path in " + + presence_name + + " %}" + "{% if path != '" + + presence_path + "' and not path.startswith('" + presence_path + + "/') %}" + "{% set " + + state_name + ".unused = " + state_name + + ".paths.append(path) %}{% endif %}{% endfor %}"; + if (is_defined_literal(assignment->value_expression)) { + block += "{% set " + state_name + ".unused = " + state_name + + ".paths.append('" + presence_path + "') %}"; + } else if (is_context_path(assignment->value_expression)) { + const std::string source_path = + presence_path_for_context_path(assignment->value_expression); + block += "{% for path in " + presence_name + + " %}" + "{% if path == '" + + source_path + "' %}{% set " + state_name + + ".unused = " + state_name + ".paths.append('" + + presence_path + "') %}{% elif path.startswith('" + + source_path + + "/') %}" + "{% set " + + state_name + ".unused = " + state_name + ".paths.append('" + + presence_path + "' + path.replace('" + source_path + + "', '', 1)) %}{% endif %}{% endfor %}"; + } + block += "{% set " + presence_name + " = " + state_name + ".paths" + + (trim_trailing_whitespace ? " -%}" : " %}"); + } + } + chat_template.replace( + block_pos, close_pos + close.size() - block_pos, block); + search_pos = block_pos + block.size(); + } + return chat_template; +} + } // namespace JinjaChatTemplate::JinjaChatTemplate(const TokenizerArgs& args) : args_(args) { try { template_ = std::make_unique( - args_.chat_template(), args_.bos_token(), args_.eos_token()); + normalize_minja_tests(args_.chat_template()), + args_.bos_token(), + args_.eos_token()); LOG(INFO) << "Jinja chat template init succeed."; } catch (const std::exception& e) { @@ -267,11 +727,28 @@ std::optional JinjaChatTemplate::apply( nlohmann::ordered_json& messages, const nlohmann::ordered_json& tools, const nlohmann::ordered_json& chat_template_kwargs) const { + if (!chat_template_kwargs.is_object()) { + LOG(ERROR) << "chat_template_kwargs must be a JSON object."; + return std::nullopt; + } + if (chat_template_kwargs.contains(kChatTemplateKwargsPresence) || + chat_template_kwargs.contains(kChatTemplateKwargsPresenceState)) { + LOG(ERROR) << "chat_template_kwargs contains reserved key: " + << (chat_template_kwargs.contains(kChatTemplateKwargsPresence) + ? kChatTemplateKwargsPresence + : kChatTemplateKwargsPresenceState); + return std::nullopt; + } + minja::chat_template_inputs input; input.messages = messages; input.tools = tools; input.add_generation_prompt = true; - input.extra_context = chat_template_kwargs; + nlohmann::ordered_json extra_context = chat_template_kwargs; + nlohmann::ordered_json extra_context_keys = nlohmann::json::array(); + append_present_paths(chat_template_kwargs, "", extra_context_keys); + extra_context[kChatTemplateKwargsPresence] = std::move(extra_context_keys); + input.extra_context = std::move(extra_context); minja::chat_template_options options; // minja throws std::runtime_error on any rendering failure (e.g. calling