diff --git a/Makefile b/Makefile index 089bcd76de..3b47a222f9 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ DS4_LINK_LIBS ?= $(CUDA_LDLIBS) METAL_LDLIBS := $(LDLIBS) endif -.PHONY: all help clean test test-metal-session-batch test-mxfp4-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm +.PHONY: all help clean test responses-replay-bench test-metal-session-batch test-mxfp4-cuda test-cuda-session-batch test-cuda-mixed-batch dspark-acceptance dspark-verify-depth mtp-verify-depth cpu cuda cuda-spark cuda-generic cuda-regression strix-halo rocm ifeq ($(UNAME_S),Darwin) .PHONY: metal-decode-schedule-bench metal-prefill-variant-bench check-mxfp4-half-lut @@ -433,6 +433,15 @@ else $(NVCC) $(NVCCFLAGS) -o $@ ds4_test.o ds4_help.o ds4_kvstore.o rax.o $(CORE_OBJS) $(CUDA_LDLIBS) endif +tests/responses_replay_bench.o: tests/responses_replay_bench.c ds4_server.c ds4.h ds4_ssd.h ds4_distributed.h ds4_help.h ds4_kvstore.h rax.h + $(CC) $(CFLAGS) -DDS4_NO_GPU -DDS4_TEST_HOOKS -Wno-unused-function -c -o $@ $< + +tests/responses_replay_bench: tests/responses_replay_bench.o ds4_help.o ds4_kvstore.o rax.o ds4_cpu_test_hooks.o $(filter-out ds4_cpu.o,$(CPU_CORE_OBJS)) + $(CC) $(CFLAGS) -o $@ $^ $(LDLIBS) + +responses-replay-bench: tests/responses_replay_bench + ./tests/responses_replay_bench + ds4_agent_test: ds4_agent_test.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) ifeq ($(UNAME_S),Darwin) $(CC) $(CFLAGS) -o $@ ds4_agent_test.o ds4_help.o ds4_web.o ds4_kvstore.o linenoise.o $(CORE_OBJS) $(METAL_LDLIBS) diff --git a/README.md b/README.md index 0f5324dec5..7662f3c8b4 100644 --- a/README.md +++ b/README.md @@ -1058,6 +1058,23 @@ and `reasoning`. It is the preferred endpoint for Codex CLI. The server keeps Responses continuations bound to live state when possible, and can fall back to the same DSML rendering and KV prefix reuse used by chat completions. +Each completed Responses turn has an opaque `resp_` id. To continue that exact +turn, send it as `previous_response_id` and put only the new user or tool-output +items in `input`. DS4 resolves the id to its saved model/KV frontier, validates +prompt-affecting settings and tool call IDs, and renders/tokenizes only the new +tail. The in-memory index is FIFO-bounded (4096 ids by default; tune it with +`--response-state-max-ids`). With disk KV caching enabled, DS4 writes an exact +post-response frontier under its opaque id only when that frontier crosses the +normal continued-cache cadence. This avoids a synchronous full KV copy after +every short turn; ids between retained boundaries are live-process-only and +cannot survive a restart or session displacement. The checkpoint persists +configuration and tool-call bindings, so a retained compatible id can be +rebuilt after a server restart or session displacement. Disk budget eviction, +an oversized checkpoint, +an incompatible model/context, changed instructions/tools/reasoning/model, or +an edited branch still requires full input replay. `conversation` objects are +not implemented. + `/v1/messages` is the Anthropic-compatible endpoint used by Claude Code style clients. It accepts `system`, `messages`, `tools`, `tool_choice`, `max_tokens`, `temperature`, `top_p`, `top_k`, `stream`, `stop_sequences`, and thinking @@ -1343,7 +1360,10 @@ Tool calls also keep a bounded exact-DSML replay map keyed by unguessable tool IDs, so client JSON history can be rendered back to the exact sampled text. The RAM map keeps up to 100000 IDs by default; tune it with `--tool-memory-max-ids`. Use `--disable-exact-dsml-tool-replay` to disable this and fall back to -canonical JSON-to-DSML rendering. +canonical JSON-to-DSML rendering. Responses response-id checkpoints use an +opaque id as their disk key rather than a visible prompt copy. Their trailer +persists the bounded continuation metadata needed to validate and resume that +checkpoint after restart. On disk, a cache file is: @@ -1352,7 +1372,7 @@ KVC fixed header, 48 bytes u32 rendered_text_bytes rendered_text_bytes of UTF-8-ish token text DS4 session payload, payload_bytes from the KVC header -optional tool-id map section +optional response-id continuation section followed by an optional tool-id map ``` The fixed header is little-endian: @@ -1362,7 +1382,9 @@ The fixed header is little-endian: 3 u8 version = 1 4 u8 routed expert quant bits, currently 2 or 4 5 u8 save reason: 0 unknown, 1 cold, 2 continued, 3 evict, 4 shutdown -6 u8 extension flags, bit 0 = appended tool-id map +6 u8 extension flags: bit 0 = appended tool-id map; bit 1 = Responses-visible key; + bit 2 = thinking-visible key; bit 3 = session-title metadata; + bit 4 = opaque response-id key 7 u8 reserved 8 u32 cached token count 12 u32 hit count @@ -1373,12 +1395,15 @@ The fixed header is little-endian: 40 u64 DS4 session payload byte count ``` -The rendered text is the tokenizer-decoded text for the cached token prefix. -It is both the human-inspectable prefix and the lookup identity: its SHA1 is -the filename, and a file is reusable only when those bytes are a prefix of the -incoming rendered prompt. After load, the exact checkpoint tokens from the DS4 -payload remain authoritative, and only the incoming text suffix after the cached -bytes is tokenized. +For ordinary entries, the rendered text is the tokenizer-decoded text for the +cached token prefix. It is both the human-inspectable prefix and the lookup +identity: its SHA1 is the filename, and a file is reusable only when those bytes +are a prefix of the incoming rendered prompt. Response-id entries instead store +their opaque server-local key as the lookup text and carry their continuation +configuration and tool-call bindings in the trailer. They are accepted only by +the response-state continuation path. After load, the exact checkpoint tokens from +the DS4 payload remain authoritative, and only the incoming text suffix after +the cached bytes is tokenized. The optional tool-id map is present only when header extension bit 0 is set. Appended sections use fixed bit order, so future extension bits can add fields @@ -1482,6 +1507,7 @@ tokens. - `--kv-cache-boundary-trim-tokens` - `--kv-cache-boundary-align-tokens` - `--tool-memory-max-ids` +- `--response-state-max-ids` - `--disable-exact-dsml-tool-replay` By default, checkpoints may be reused across the 2-bit and 4-bit routed-expert @@ -1489,8 +1515,9 @@ variants if the rendered prefix matches. Use `--kv-cache-reject-different-quant` when you want strict same-quant reuse only. The cache directory is disposable. If behavior looks suspicious, stop the -server and remove it. You can investigate what is cached with hexdump as -the kv cache files include the verbatim prompt cached. +server and remove it. You can investigate what is cached with hexdump: ordinary +entries include their rendered prompt, while response-id entries use an opaque +server-local checkpoint key. ## Backends diff --git a/ds4.c b/ds4.c index 449140b523..a04243042e 100644 --- a/ds4.c +++ b/ds4.c @@ -36903,6 +36903,11 @@ struct ds4_engine { * caller that doesn't set the option observe the prior behavior). */ int placement_ctx_hint; int placement_session_count_hint; +#ifdef DS4_TEST_HOOKS + /* Test tokenizer fixtures have no routed tensors but still need to drive + * disk-KV policy that is normally selected from the model quantization. */ + int test_routed_quant_bits; +#endif }; static uint64_t ds4_engine_dynamic_expert_cache_bytes( @@ -37911,6 +37916,61 @@ void ds4_tokenize_rendered_chat(ds4_engine *e, const char *text, ds4_tokens *out tokenize_rendered_chat_vocab(&e->vocab, text, out); } +#ifdef DS4_TEST_HOOKS +/* Build only the tokenizer state needed by server/parser tests. Each raw byte + * maps to its GPT-2 byte-encoding token, while synthetic special-token IDs make + * rendered chat delimiters take the same fast path as a loaded model tokenizer. */ +ds4_engine *ds4_test_engine_create_byte_tokenizer(void) { + enum { TEST_BYTE_TOKENS = 256, TEST_SPECIAL_FIRST = 256 }; + ds4_engine *e = xcalloc(1, sizeof(*e)); + e->backend = DS4_BACKEND_CPU; + e->test_routed_quant_bits = 2; + ds4_vocab *vocab = &e->vocab; + vocab->n_vocab = TEST_SPECIAL_FIRST + 20; + vocab->token = xcalloc((size_t)vocab->n_vocab, sizeof(vocab->token[0])); + table_init(&vocab->token_to_id, TEST_BYTE_TOKENS); + table_init(&vocab->merge_rank, 0); + for (int i = 0; i < TEST_BYTE_TOKENS; i++) { + char encoded[5]; + char *end = encoded; + utf8_put(&end, gpt2_byte_to_codepoint((uint8_t)i)); + size_t len = (size_t)(end - encoded); + char *owned = xmalloc(len + 1); + memcpy(owned, encoded, len); + owned[len] = '\0'; + vocab->token[i] = (ds4_str){.ptr = owned, .len = len}; + table_put(&vocab->token_to_id, vocab->token[i], i); + } + int special = TEST_SPECIAL_FIRST; + vocab->bos_id = special++; + vocab->eos_id = special++; + vocab->system_id = special++; + vocab->user_id = special++; + vocab->assistant_id = special++; + vocab->observation_id = special++; + vocab->sop_id = special++; + vocab->think_start_id = special++; + vocab->think_end_id = special++; + vocab->tool_call_start_id = special++; + vocab->tool_call_end_id = special++; + vocab->tool_response_start_id = special++; + vocab->tool_response_end_id = special++; + vocab->arg_key_start_id = special++; + vocab->arg_key_end_id = special++; + vocab->arg_value_start_id = special++; + vocab->arg_value_end_id = special++; + vocab->dsml_id = special++; + return e; +} + +void ds4_test_engine_free_byte_tokenizer(ds4_engine *e) { + if (!e) return; + for (int i = 0; i < 256; i++) free((void *)e->vocab.token[i].ptr); + vocab_free(&e->vocab); + free(e); +} +#endif + void ds4_chat_begin(ds4_engine *e, ds4_tokens *tokens) { chat_push_bos_sequence(&e->vocab, tokens); } @@ -49336,8 +49396,35 @@ struct ds4_session { bool checkpoint_valid; bool mtp_draft_valid; bool greedy_splitkv_anchor_valid; +#ifdef DS4_TEST_HOOKS + bool test_token_only; + uint64_t test_payload_bytes; + uint64_t test_payload_save_count; +#endif }; +#ifdef DS4_TEST_HOOKS +ds4_session *ds4_test_session_create_token_only(ds4_engine *e, int ctx_size) { + if (!e || ctx_size <= 1) return NULL; + ds4_session *s = xcalloc(1, sizeof(*s)); + s->engine = e; + s->ctx_size = ctx_size; + s->prefill_cap = (uint32_t)ctx_size; + s->test_token_only = true; + return s; +} + +bool ds4_test_session_set_payload_bytes(ds4_session *s, uint64_t bytes) { + if (!s || !s->test_token_only) return false; + s->test_payload_bytes = bytes; + return true; +} + +uint64_t ds4_test_session_payload_save_count(const ds4_session *s) { + return s && s->test_token_only ? s->test_payload_save_count : 0; +} +#endif + #ifndef DS4_NO_GPU static bool ds4_dspark_stats_enabled(void); @@ -50943,6 +51030,10 @@ int ds4_session_load_layer_payload(ds4_session *s, FILE *fp, int ds4_engine_routed_quant_bits(ds4_engine *e) { if (!e) return 0; +#ifdef DS4_TEST_HOOKS + if (e->test_routed_quant_bits == 2 || e->test_routed_quant_bits == 4) + return e->test_routed_quant_bits; +#endif for (uint32_t il = 0; il < DS4_N_LAYER; il++) { const ds4_tensor *gate = e->weights.layer[il].ffn_gate_exps; if (!gate) continue; @@ -51123,8 +51214,115 @@ static void session_greedy_splitkv_reset(ds4_session *s) { } #endif +#ifdef DS4_TEST_HOOKS +/* The server ingress fixture needs a real restartable payload so its disk-cache + * path copies the same byte volume as a production checkpoint. Keep this + * format private to the token-only fixture: it preserves exact tokens but has + * no model tensors or logits. */ +#define DS4_TEST_TOKEN_ONLY_PAYLOAD_MAGIC UINT32_C(0x54534b56) +#define DS4_TEST_TOKEN_ONLY_PAYLOAD_VERSION UINT32_C(1) +#define DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS 4u + +static uint64_t ds4_test_token_only_payload_size(const ds4_session *s) { + if (!s || !s->test_token_only || !s->checkpoint_valid || + s->checkpoint.len < 0) + return 0; + const uint64_t token_bytes = (uint64_t)s->checkpoint.len * sizeof(uint32_t); + const uint64_t header_bytes = + (uint64_t)DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS * sizeof(uint32_t); + if (token_bytes > UINT64_MAX - header_bytes) return 0; + const uint64_t minimum = header_bytes + token_bytes; + return s->test_payload_bytes > minimum ? s->test_payload_bytes : minimum; +} + +static int ds4_test_save_token_only_payload(ds4_session *s, FILE *fp, + char *err, size_t errlen) { + const uint64_t payload_bytes = ds4_test_token_only_payload_size(s); + if (!payload_bytes || !fp || s->checkpoint.len >= s->ctx_size) { + payload_set_err(err, errlen, "invalid token-only session payload"); + return 1; + } + s->test_payload_save_count++; + const uint32_t header[DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS] = { + DS4_TEST_TOKEN_ONLY_PAYLOAD_MAGIC, + DS4_TEST_TOKEN_ONLY_PAYLOAD_VERSION, + (uint32_t)s->ctx_size, + (uint32_t)s->checkpoint.len, + }; + for (uint32_t i = 0; i < DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS; i++) { + if (payload_write_u32(fp, header[i], err, errlen) != 0) return 1; + } + for (int i = 0; i < s->checkpoint.len; i++) { + if (payload_write_u32(fp, (uint32_t)s->checkpoint.v[i], err, errlen) != 0) + return 1; + } + + const uint64_t written = + (uint64_t)DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS * sizeof(uint32_t) + + (uint64_t)s->checkpoint.len * sizeof(uint32_t); + static const uint8_t zeroes[4096]; + uint64_t padding = payload_bytes - written; + while (padding != 0) { + const size_t n = padding > sizeof(zeroes) ? sizeof(zeroes) : (size_t)padding; + if (payload_write_bytes(fp, zeroes, n, err, errlen) != 0) return 1; + padding -= n; + } + return 0; +} + +static int ds4_test_load_token_only_payload(ds4_session *s, FILE *fp, + uint64_t payload_bytes, + char *err, size_t errlen) { + if (!s || !fp || payload_bytes < + (uint64_t)DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS * sizeof(uint32_t)) + { + payload_set_err(err, errlen, "truncated token-only session payload"); + return 1; + } + uint64_t remaining = payload_bytes; + uint32_t header[DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS]; + for (uint32_t i = 0; i < DS4_TEST_TOKEN_ONLY_PAYLOAD_U32_FIELDS; i++) { + if (payload_read_u32(fp, &header[i], &remaining, err, errlen) != 0) return 1; + } + const uint32_t saved_tokens = header[3]; + const uint64_t token_bytes = (uint64_t)saved_tokens * sizeof(uint32_t); + if (header[0] != DS4_TEST_TOKEN_ONLY_PAYLOAD_MAGIC || + header[1] != DS4_TEST_TOKEN_ONLY_PAYLOAD_VERSION || + header[2] > (uint32_t)s->ctx_size || + saved_tokens >= (uint32_t)s->ctx_size || token_bytes > remaining) + { + payload_set_err(err, errlen, "invalid token-only session payload"); + return 1; + } + + s->checkpoint.len = 0; + for (uint32_t i = 0; i < saved_tokens; i++) { + uint32_t token = 0; + if (payload_read_u32(fp, &token, &remaining, err, errlen) != 0) return 1; + if (!s->engine || token >= (uint32_t)s->engine->vocab.n_vocab) { + payload_set_err(err, errlen, "token-only payload token is outside vocabulary"); + return 1; + } + token_vec_push(&s->checkpoint, (int)token); + } + if (remaining != 0) { + uint8_t discard[4096]; + if (payload_skip_bytes(fp, remaining, discard, sizeof(discard), + &remaining, err, errlen) != 0) + return 1; + } + s->test_payload_bytes = payload_bytes; + s->checkpoint_valid = true; + s->mtp_draft_valid = false; + return 0; +} +#endif + uint64_t ds4_session_payload_bytes(ds4_session *s) { if (!s || !s->checkpoint_valid) return 0; +#ifdef DS4_TEST_HOOKS + if (s->test_token_only) return ds4_test_token_only_payload_size(s); +#endif if (s->distributed) return 0; if (ds4_session_is_cpu(s)) { uint64_t bytes = (uint64_t)DS4_SESSION_PAYLOAD_U32_FIELDS * sizeof(uint32_t); @@ -51255,6 +51453,9 @@ int ds4_session_save_payload(ds4_session *s, FILE *fp, char *err, size_t errlen) payload_set_err(err, errlen, "session has no valid checkpoint to save"); return 1; } +#ifdef DS4_TEST_HOOKS + if (s->test_token_only) return ds4_test_save_token_only_payload(s, fp, err, errlen); +#endif if (s->distributed) { return ds4_dist_session_save_payload(s->distributed, s, fp, err, errlen); } @@ -51564,6 +51765,10 @@ int ds4_session_load_payload(ds4_session *s, FILE *fp, uint64_t payload_bytes, c payload_set_err(err, errlen, "invalid session payload load"); return 1; } +#ifdef DS4_TEST_HOOKS + if (s->test_token_only) + return ds4_test_load_token_only_payload(s, fp, payload_bytes, err, errlen); +#endif if (s->distributed) { return ds4_dist_session_load_payload(s->distributed, s, fp, payload_bytes, err, errlen); } @@ -59836,6 +60041,22 @@ static int ds4_session_sync_internal(ds4_session *s, const ds4_tokens *prompt, c snprintf(err, errlen, "interrupted"); return DS4_SESSION_SYNC_INTERRUPTED; } +#ifdef DS4_TEST_HOOKS + if (s->test_token_only) { + if (s->checkpoint_valid && prompt->len >= s->checkpoint.len && + ds4_tokens_starts_with(prompt, &s->checkpoint)) + { + for (int i = s->checkpoint.len; i < prompt->len; i++) { + token_vec_push(&s->checkpoint, prompt->v[i]); + } + } else { + ds4_tokens_copy(&s->checkpoint, prompt); + } + s->checkpoint_valid = true; + if (s->progress) s->progress(s->progress_ud, "prefill_chunk", prompt->len, prompt->len); + return 0; + } +#endif if (s->distributed) { const ds4_tokens *checkpoint = s->checkpoint_valid ? &s->checkpoint : NULL; return ds4_dist_session_sync(s->distributed, @@ -60743,6 +60964,9 @@ int ds4_sample_logits(const float *logits, int n_vocab, float temperature, } int ds4_session_sample(ds4_session *s, float temperature, int top_k, float top_p, float min_p, uint64_t *rng) { +#ifdef DS4_TEST_HOOKS + if (s && s->test_token_only) return 'A'; +#endif return sample_top_p_min_p(s->logits, DS4_N_VOCAB, temperature, top_k, top_p, min_p, rng, s->sample_probs); } @@ -61462,6 +61686,18 @@ static void ds4_session_prepare_support_draft(ds4_session *s, static int ds4_session_eval_internal(ds4_session *s, int token, bool probe_mtp, char *err, size_t errlen) { if (!s) return 1; +#ifdef DS4_TEST_HOOKS + if (s->test_token_only) { + if (s->checkpoint.len >= s->ctx_size - 1) { + if (errlen) snprintf(err, errlen, "test session context reached"); + return 1; + } + token_vec_push(&s->checkpoint, token); + s->checkpoint_valid = true; + (void)probe_mtp; + return 0; + } +#endif if (s->distributed) { if (!s->checkpoint_valid) { if (errlen) snprintf(err, errlen, "distributed decode requires a valid checkpoint"); diff --git a/ds4.h b/ds4.h index d9c7e2f62c..92884a9d53 100644 --- a/ds4.h +++ b/ds4.h @@ -376,6 +376,18 @@ int ds4_test_sample_logits(const float *logits, uint32_t n_vocab, int ds4_test_argmax_excluding_logits(const float *logits, uint32_t n_vocab, int excluded_id); uint64_t ds4_test_mixed_native_count(void); +/* A tokenizer-only engine with a synthetic byte vocabulary. It exercises the + * production rendered-chat tokenizer without loading model weights. */ +ds4_engine *ds4_test_engine_create_byte_tokenizer(void); +void ds4_test_engine_free_byte_tokenizer(ds4_engine *e); +/* A checkpoint-only session for server integration tests. Its sync path keeps + * the exact token prefix but intentionally performs no model evaluation. */ +ds4_session *ds4_test_session_create_token_only(ds4_engine *e, int ctx_size); +/* Pad this test-only session's restartable checkpoint to exercise disk-copy + * paths without model weights. The exact token prefix remains authoritative. */ +bool ds4_test_session_set_payload_bytes(ds4_session *s, uint64_t bytes); +/* Number of attempts to serialize this fixture's checkpoint payload. */ +uint64_t ds4_test_session_payload_save_count(const ds4_session *s); #endif int ds4_session_top_logprobs(ds4_session *s, ds4_token_score *out, int k); int ds4_session_token_logprob(ds4_session *s, int token, ds4_token_score *out); diff --git a/ds4_help.c b/ds4_help.c index adc8e634e8..81b7ea39f8 100644 --- a/ds4_help.c +++ b/ds4_help.c @@ -361,6 +361,7 @@ static void print_kv_cache(FILE *fp, const help_colors *c) { opt(fp, c, "--kv-cache-reject-different-quant", "Reject checkpoints written with different routed-expert quantization."); opt(fp, c, "--disable-exact-dsml-tool-replay", "Disable exact sampled DSML tool replay map."); opt(fp, c, "--tool-memory-max-ids N", "Exact tool-call IDs kept in RAM. Default: 100000"); + opt(fp, c, "--response-state-max-ids N", "Completed Responses response IDs retained locally. Default: 4096"); fputc('\n', fp); } diff --git a/ds4_kvstore.c b/ds4_kvstore.c index 5b73de7b65..504421025f 100644 --- a/ds4_kvstore.c +++ b/ds4_kvstore.c @@ -183,6 +183,7 @@ uint8_t ds4_kvstore_reason_code(const char *reason) { } const char *ds4_kvstore_key_kind(uint8_t ext_flags) { + if (ext_flags & DS4_KVSTORE_EXT_RESPONSES_ID) return "responses-id"; if (ext_flags & DS4_KVSTORE_EXT_RESPONSES_VISIBLE) return "responses-visible"; if (ext_flags & DS4_KVSTORE_EXT_THINKING_VISIBLE) return "thinking-visible"; return "token-text"; @@ -738,13 +739,21 @@ static int kv_cache_continued_step(const ds4_kvstore *kc) { return step; } -int ds4_kvstore_continued_store_target(const ds4_kvstore *kc, int live_tokens) { +int ds4_kvstore_continued_store_crossed_target(const ds4_kvstore *kc, + int last_store_tokens, + int live_tokens) { + if (!kc) return 0; const int step = kv_cache_continued_step(kc); - if (step <= 0) return 0; - if (live_tokens < kc->opt.min_tokens) return 0; - if (live_tokens % step != 0) return 0; - if (live_tokens <= kc->continued_last_store_tokens) return 0; - return live_tokens; + if (step <= 0 || live_tokens < kc->opt.min_tokens) return 0; + const int boundary = live_tokens - live_tokens % step; + if (boundary < kc->opt.min_tokens || boundary <= last_store_tokens) return 0; + return boundary; +} + +int ds4_kvstore_continued_store_target(const ds4_kvstore *kc, int live_tokens) { + const int boundary = ds4_kvstore_continued_store_crossed_target( + kc, kc ? kc->continued_last_store_tokens : 0, live_tokens); + return boundary == live_tokens ? boundary : 0; } void ds4_kvstore_note_store(ds4_kvstore *kc, int tokens) { @@ -925,6 +934,7 @@ bool ds4_kvstore_store_live_prefix_text(ds4_kvstore *kc, ds4_session *session, const ds4_tokens *tokens, int store_len, + bool force_store, const char *reason, const char *cache_text_override, uint8_t cache_text_ext, @@ -933,7 +943,7 @@ bool ds4_kvstore_store_live_prefix_text(ds4_kvstore *kc, char *err, size_t err_len) { if (!kc->enabled) return false; - if (!tokens || store_len < kc->opt.min_tokens) return false; + if (!tokens || (!force_store && store_len < kc->opt.min_tokens)) return false; const int original_len = tokens->len; ds4_tokens store_tokens = {0}; @@ -1164,7 +1174,7 @@ bool ds4_kvstore_store_live_prefix(ds4_kvstore *kc, char *err, size_t err_len) { return ds4_kvstore_store_live_prefix_text(kc, engine, session, tokens, - store_len, reason, NULL, 0, NULL, + store_len, false, reason, NULL, 0, NULL, hooks, err, err_len); } @@ -1219,7 +1229,8 @@ int ds4_kvstore_try_load_text(ds4_kvstore *kc, ds4_tokens *effective_prompt, ds4_kvstore_load_result *result, const ds4_kvstore_trailer_hooks *hooks, - bool responses_protocol) { + bool responses_protocol, + uint8_t required_ext_flags) { if (result) memset(result, 0, sizeof(*result)); if (effective_prompt) effective_prompt->len = 0; if (!kc->enabled || !prompt_text) return 0; @@ -1248,6 +1259,10 @@ int ds4_kvstore_try_load_text(ds4_kvstore *kc, if (hdr.model_id != (uint8_t)model_id) { header_ok = false; fail_reason = "cached checkpoint was written for a different model"; + } else if (required_ext_flags && + (hdr.ext_flags & required_ext_flags) != required_ext_flags) { + header_ok = false; + fail_reason = "cached checkpoint has incompatible extension flags"; } else if ((uint64_t)text_bytes > prompt_bytes) { header_ok = false; fail_reason = "cached text is longer than prompt"; diff --git a/ds4_kvstore.h b/ds4_kvstore.h index 28ccdb7eaf..48e585046a 100644 --- a/ds4_kvstore.h +++ b/ds4_kvstore.h @@ -16,6 +16,8 @@ #define DS4_KVSTORE_EXT_RESPONSES_VISIBLE (1u << 1) #define DS4_KVSTORE_EXT_THINKING_VISIBLE (1u << 2) #define DS4_KVSTORE_EXT_SESSION_TITLE (1u << 3) +/* Exact checkpoint keyed by a server-local /v1/responses response id. */ +#define DS4_KVSTORE_EXT_RESPONSES_ID (1u << 4) typedef enum { DS4_KVSTORE_REASON_UNKNOWN = 0, @@ -137,6 +139,12 @@ int ds4_kvstore_chat_anchor_pos(const ds4_kvstore *kc, int user_token_id, int assistant_token_id); int ds4_kvstore_continued_store_target(const ds4_kvstore *kc, int live_tokens); +/* Return the latest continued-cache boundary crossed by live_tokens after + * last_store_tokens, or zero when no new boundary is due. This lets callers + * that need an exact post-boundary snapshot share the normal cache cadence. */ +int ds4_kvstore_continued_store_crossed_target(const ds4_kvstore *kc, + int last_store_tokens, + int live_tokens); void ds4_kvstore_note_store(ds4_kvstore *kc, int tokens); int ds4_kvstore_suppress_continued_store(ds4_kvstore *kc, int tokens); void ds4_kvstore_restore_suppressed_continued(ds4_kvstore *kc, @@ -164,6 +172,8 @@ bool ds4_kvstore_store_live_prefix_text(ds4_kvstore *kc, ds4_session *session, const ds4_tokens *tokens, int store_len, + /* Bypass only the minimum-prefix policy; disk budget still applies. */ + bool force_store, const char *reason, const char *cache_text_override, uint8_t cache_text_ext, @@ -186,6 +196,8 @@ bool ds4_kvstore_maybe_store_continued(ds4_kvstore *kc, const ds4_kvstore_trailer_hooks *hooks, char *err, size_t err_len); +/* required_ext_flags rejects a text-key match before it can replace the live + * session with a checkpoint from a different protocol namespace. */ int ds4_kvstore_try_load_text(ds4_kvstore *kc, ds4_engine *engine, ds4_session *session, @@ -193,7 +205,8 @@ int ds4_kvstore_try_load_text(ds4_kvstore *kc, ds4_tokens *effective_prompt, ds4_kvstore_load_result *result, const ds4_kvstore_trailer_hooks *hooks, - bool responses_protocol); + bool responses_protocol, + uint8_t required_ext_flags); void ds4_kvstore_load_result_free(ds4_kvstore_load_result *result); bool ds4_kvstore_read_header(FILE *fp, ds4_kvstore_entry *e, diff --git a/ds4_server.c b/ds4_server.c index 0ed35d4d39..ddccbef441 100644 --- a/ds4_server.c +++ b/ds4_server.c @@ -565,6 +565,11 @@ static void random_tool_id(char *dst, size_t dstlen, api_style api) { } typedef struct server server; +typedef struct response_state_entry response_state_entry; + +/* A parsed request holds a pinned response-state record while it waits in the + * server queue. The record itself lives in the bounded server-local index. */ +static void response_state_release(server *s, response_state_entry *entry); typedef struct { char *id; @@ -693,12 +698,50 @@ typedef struct { bool responses_requires_live_reasoning; stop_list responses_live_call_ids; char *responses_live_suffix_text; + /* previous_response_id continuation. On a state hit prompt contains only + * the rendered tail; generate_job() joins it to the saved exact frontier. */ + char *responses_previous_id; + char *responses_state_tail_text; + char *responses_state_instructions; + char *responses_state_request_tools; + char *responses_state_active_tools; + bool responses_state_tool_choice_none; + bool responses_state_instructions_set; + bool responses_state_request_tools_set; + bool responses_state_tool_choice_set; + bool responses_state_thinking_set; + uint64_t responses_state_fingerprint; + response_state_entry *responses_state; + server *responses_state_server; + bool responses_stateful; bool anthropic_requires_live_tool_state; stop_list anthropic_live_call_ids; char *anthropic_live_suffix_text; tool_replay_stats tool_replay; } request; +static uint64_t response_state_fingerprint_values(server_model_syntax syntax, + ds4_think_mode think_mode, + bool has_tools, + bool tool_choice_none, + const char *model, + const char *instructions, + const char *request_tools, + const char *active_tools); +static response_state_entry *response_state_acquire(server *s, const char *id); +/* If an optional KV checkpoint survived a process restart, rebuild its bounded + * response-id record before treating the request as a replay fallback. */ +static response_state_entry *response_state_acquire_or_restore(server *s, const char *id); +static bool response_state_config_matches(const response_state_entry *entry, + const request *r); +static const stop_list *response_state_call_ids(const response_state_entry *entry); +static bool responses_input_is_append_only(const chat_msgs *msgs); +static bool responses_validate_state_tool_outputs(const chat_msgs *msgs, + const stop_list *known_ids, + char *err, size_t errlen); +static void response_state_apply(request *r, response_state_entry *entry, + const char *delta_tools); + static void tool_call_free(tool_call *tc) { free(tc->id); free(tc->name); @@ -810,6 +853,30 @@ static const tool_schema_order *tool_schema_orders_find(const tool_schema_orders return idx >= 0 ? &orders->v[idx] : NULL; } +/* Response-state records outlive the request that introduced a tool schema. + * Keep an owned copy of the ordered schema metadata so a later function-call + * continuation renders its arguments in the same wire order. */ +static tool_schema_order tool_schema_order_clone(const tool_schema_order *src) { + tool_schema_order dst = {0}; + if (!src) return dst; + dst.name = src->name ? xstrdup(src->name) : NULL; + dst.wire_name = src->wire_name ? xstrdup(src->wire_name) : NULL; + dst.namespace = src->namespace ? xstrdup(src->namespace) : NULL; + dst.responses_tool_search = src->responses_tool_search; + for (int i = 0; i < src->len; i++) { + tool_schema_order_prop_push(&dst, xstrdup(src->prop[i] ? src->prop[i] : "")); + } + return dst; +} + +static void tool_schema_orders_clone_append(tool_schema_orders *dst, + const tool_schema_orders *src) { + if (!dst || !src) return; + for (int i = 0; i < src->len; i++) { + tool_schema_orders_push(dst, tool_schema_order_clone(&src->v[i])); + } +} + static void request_init(request *r, req_kind kind, int max_tokens) { memset(r, 0, sizeof(*r)); r->kind = kind; @@ -834,6 +901,12 @@ static void request_free(request *r) { stop_list_clear(&r->responses_live_call_ids); free(r->responses_live_call_ids.v); free(r->responses_live_suffix_text); + free(r->responses_previous_id); + free(r->responses_state_tail_text); + free(r->responses_state_instructions); + free(r->responses_state_request_tools); + free(r->responses_state_active_tools); + if (r->responses_state) response_state_release(r->responses_state_server, r->responses_state); stop_list_clear(&r->anthropic_live_call_ids); free(r->anthropic_live_call_ids.v); free(r->anthropic_live_suffix_text); @@ -841,6 +914,21 @@ static void request_free(request *r) { memset(r, 0, sizeof(*r)); } +static void responses_request_set_state_config(request *r, + const char *instructions, + const char *request_tools, + const char *active_tools, + bool tool_choice_none) { + if (!r) return; + free(r->responses_state_instructions); + free(r->responses_state_request_tools); + free(r->responses_state_active_tools); + r->responses_state_instructions = xstrdup(instructions ? instructions : ""); + r->responses_state_request_tools = xstrdup(request_tools ? request_tools : ""); + r->responses_state_active_tools = xstrdup(active_tools ? active_tools : ""); + r->responses_state_tool_choice_none = tool_choice_none; +} + static ds4_think_mode think_mode_from_enabled(bool enabled, ds4_think_mode effort) { if (!enabled || effort == DS4_THINK_NONE) return DS4_THINK_NONE; return effort == DS4_THINK_MAX ? DS4_THINK_MAX : DS4_THINK_HIGH; @@ -4047,8 +4135,11 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, ds4_think_mode reasoning_effort = DS4_THINK_HIGH; chat_msgs msgs = {0}; buf loaded_tool_schemas = {0}; + buf combined_tool_schemas = {0}; char *instructions = NULL; char *tool_schemas = NULL; + char *previous_response_id = NULL; + response_state_entry *state = NULL; json_ws(&p); if (*p != '{') goto bad; @@ -4085,6 +4176,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, } got_input = true; } else if (!strcmp(key, "instructions")) { + r->responses_state_instructions_set = true; free(instructions); instructions = NULL; json_ws(&p); @@ -4095,6 +4187,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, goto bad; } } else if (!strcmp(key, "tools")) { + r->responses_state_request_tools_set = true; free(tool_schemas); tool_schemas = NULL; if (!parse_tools_value(&p, &tool_schemas, &r->tool_orders)) { @@ -4102,6 +4195,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, goto bad; } } else if (!strcmp(key, "tool_choice")) { + r->responses_state_tool_choice_set = true; json_ws(&p); if (*p == '"') { char *choice = NULL; @@ -4123,6 +4217,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); + free(previous_response_id); request_free(r); return false; } @@ -4134,6 +4229,7 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); + free(previous_response_id); request_free(r); return false; } else if (!json_skip_value(&p)) { @@ -4185,33 +4281,32 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, * default behaviour (and the model_alias_* fallbacks below) intact. */ if (effort_seen) { got_thinking = true; + r->responses_state_thinking_set = true; /* Responses-API effort of "minimal" / "none" maps to disabled * thinking. Other effort values choose between HIGH and MAX. */ if (reasoning_effort == DS4_THINK_NONE) thinking_enabled = false; } - } else if (!strcmp(key, "previous_response_id") || - !strcmp(key, "conversation")) - { - /* Official Responses state can be durable: - * previous_response_id chains to a stored prior response, and - * conversation points at a persistent Conversations object. - * - * DS4 does not yet implement that durable store. The supported - * modes are either (a) a live in-memory continuation checked by - * visible transcript / tool call ids, or (b) stateless replay of - * the full input items. Accepting a non-null durable reference - * without loading the referenced items would silently truncate the - * prompt, so reject it explicitly. */ + } else if (!strcmp(key, "previous_response_id")) { + free(previous_response_id); + previous_response_id = NULL; + json_ws(&p); + if (!json_lit(&p, "null") && !json_string(&p, &previous_response_id)) { + free(key); + goto bad; + } + } else if (!strcmp(key, "conversation")) { + /* Conversations objects need their own persistent object store; + * previous_response_id is the bounded local continuation handle. */ json_ws(&p); if (!json_lit(&p, "null")) { snprintf(err, errlen, - "%s is not supported; replay full input instead", - key); + "conversation is not supported; use previous_response_id or replay full input instead"); free(key); chat_msgs_free(&msgs); buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); + free(previous_response_id); request_free(r); return false; } @@ -4227,13 +4322,83 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, if (*p != '}') goto bad; if (!got_input) { snprintf(err, errlen, "missing input"); - chat_msgs_free(&msgs); - buf_free(&loaded_tool_schemas); - free(instructions); - free(tool_schemas); - request_free(r); - return false; + goto rejected; + } + + /* Keep the request's prompt-affecting configuration separately from its + * rendered history. A previous_response_id may inherit omitted fields + * from the saved response, but an explicit edit must not silently reuse a + * different model prefix. */ + if (tool_schemas && tool_schemas[0]) buf_puts(&combined_tool_schemas, tool_schemas); + if (loaded_tool_schemas.len) { + if (combined_tool_schemas.len) buf_putc(&combined_tool_schemas, '\n'); + buf_append(&combined_tool_schemas, loaded_tool_schemas.ptr, + loaded_tool_schemas.len); + } + const char *active_tool_schemas = + (!tool_choice_none && combined_tool_schemas.len) ? + combined_tool_schemas.ptr : NULL; + r->has_tools = active_tool_schemas && active_tool_schemas[0]; + if (!got_thinking && model_alias_disables_thinking(r->model)) thinking_enabled = false; + if (!got_thinking && model_alias_enables_thinking(r->model)) thinking_enabled = true; + r->think_mode = ds4_think_mode_for_context( + think_mode_from_enabled(thinking_enabled, reasoning_effort), ctx_size); + responses_request_set_state_config(r, instructions ? instructions : "", + tool_schemas ? tool_schemas : "", + active_tool_schemas ? active_tool_schemas : "", + tool_choice_none); + r->responses_state_fingerprint = response_state_fingerprint_values( + r->model_syntax, r->think_mode, r->has_tools, + r->responses_state_tool_choice_none, r->model, + r->responses_state_instructions, r->responses_state_request_tools, + r->responses_state_active_tools); + + if (previous_response_id && previous_response_id[0]) { + state = response_state_acquire_or_restore(s, previous_response_id); + /* Dynamic tool-search schemas are rendered at the beginning of a full + * chat prompt, so they cannot safely be inserted into a generic tail. + * Treat that rare shape as an explicit replay branch instead. */ + bool append_only = responses_input_is_append_only(&msgs) && + loaded_tool_schemas.len == 0; + if (state && append_only && response_state_config_matches(state, r)) { + if (!responses_validate_state_tool_outputs(&msgs, response_state_call_ids(state), + err, errlen)) { + goto rejected; + } + response_state_apply(r, state, NULL); + r->responses_state = state; + r->responses_state_server = s; + r->responses_stateful = true; + state = NULL; /* request now owns the acquired reference */ + r->responses_previous_id = previous_response_id; + previous_response_id = NULL; + r->responses_state_tail_text = render_live_tool_tail_for_syntax( + r->model_syntax, &msgs, 0, &r->tool_orders, r->think_mode); + r->prompt_text = xstrdup(r->responses_state_tail_text ? + r->responses_state_tail_text : ""); + ds4_tokenize_rendered_chat(e, r->prompt_text, &r->prompt); + r->prompt_preserves_reasoning = true; + chat_msgs_free(&msgs); + buf_free(&combined_tool_schemas); + buf_free(&loaded_tool_schemas); + free(instructions); + free(tool_schemas); + return true; + } + if (state) { + response_state_release(s, state); + state = NULL; + } + if (append_only) { + snprintf(err, errlen, + "previous_response_id is unavailable or its prompt configuration changed; replay the full input history"); + goto rejected; + } + /* An assistant/system item (or dynamic hosted-tool schema) means this + * is a deliberate full replay/edit. Preserve the existing stateless + * parser/render/tokenize path below. */ } + /* instructions in the Responses API replaces any system message — for Codex * it carries the full agent system prompt. Prepend it so render produces a * standard system+chat layout. */ @@ -4250,32 +4415,11 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, msgs.v[0] = tmp; } } - buf combined_tool_schemas = {0}; - if (tool_schemas && tool_schemas[0]) buf_puts(&combined_tool_schemas, tool_schemas); - if (loaded_tool_schemas.len) { - if (combined_tool_schemas.len) buf_putc(&combined_tool_schemas, '\n'); - buf_append(&combined_tool_schemas, loaded_tool_schemas.ptr, - loaded_tool_schemas.len); - } - const char *active_tool_schemas = - (!tool_choice_none && combined_tool_schemas.len) ? - combined_tool_schemas.ptr : NULL; - r->has_tools = active_tool_schemas && active_tool_schemas[0]; - if (!got_thinking && model_alias_disables_thinking(r->model)) thinking_enabled = false; - if (!got_thinking && model_alias_enables_thinking(r->model)) thinking_enabled = true; - r->think_mode = ds4_think_mode_for_context( - think_mode_from_enabled(thinking_enabled, reasoning_effort), ctx_size); if (!responses_validate_tool_outputs(s, &msgs, r->think_mode, &r->responses_requires_live_tool_state, &r->responses_requires_live_reasoning, err, errlen)) { - chat_msgs_free(&msgs); - buf_free(&combined_tool_schemas); - buf_free(&loaded_tool_schemas); - free(instructions); - free(tool_schemas); - request_free(r); - return false; + goto rejected; } kv_cache_restore_tool_memory_for_messages(s, &msgs); tool_memory_attach_to_messages(s, &msgs, &r->tool_replay); @@ -4291,12 +4435,26 @@ static bool parse_responses_request(ds4_engine *e, server *s, const char *body, buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); + free(previous_response_id); return true; +rejected: + if (state) response_state_release(s, state); + chat_msgs_free(&msgs); + buf_free(&combined_tool_schemas); + buf_free(&loaded_tool_schemas); + free(instructions); + free(tool_schemas); + free(previous_response_id); + request_free(r); + return false; bad: + if (state) response_state_release(s, state); chat_msgs_free(&msgs); + buf_free(&combined_tool_schemas); buf_free(&loaded_tool_schemas); free(instructions); free(tool_schemas); + free(previous_response_id); snprintf(err, errlen, "invalid JSON request"); request_free(r); return false; @@ -6679,10 +6837,15 @@ typedef struct { int sequence; /* monotonic per-event sequence_number Codex consumes */ } responses_stream; -static void responses_stream_init(const request *r, responses_stream *st) { +static void responses_stream_init(const request *r, responses_stream *st, + const char *response_id) { memset(st, 0, sizeof(*st)); st->mode = ds4_think_mode_enabled(r->think_mode) ? RESP_STREAM_THINKING : RESP_STREAM_TEXT; - responses_random_id(st->response_id, sizeof(st->response_id), "resp_"); + if (response_id && response_id[0]) { + snprintf(st->response_id, sizeof(st->response_id), "%s", response_id); + } else { + responses_random_id(st->response_id, sizeof(st->response_id), "resp_"); + } responses_random_id(st->reasoning_id, sizeof(st->reasoning_id), "rs_"); responses_random_id(st->message_id, sizeof(st->message_id), "msg_"); st->reasoning_index = -1; @@ -7378,13 +7541,15 @@ static bool responses_sse_finish_live(int fd, const request *r, } static bool responses_final_response(int fd, bool enable_cors, - const request *r, const char *id, + const request *r, const char *response_id, const char *text, const char *reasoning, const tool_calls *calls, const char *finish, int prompt_tokens, int completion_tokens) { - (void)id; - char response_id[40], reasoning_id[40], message_id[40]; - responses_random_id(response_id, sizeof(response_id), "resp_"); + char generated_response_id[40], reasoning_id[40], message_id[40]; + if (!response_id || !response_id[0]) { + responses_random_id(generated_response_id, sizeof(generated_response_id), "resp_"); + response_id = generated_response_id; + } responses_random_id(reasoning_id, sizeof(reasoning_id), "rs_"); responses_random_id(message_id, sizeof(message_id), "msg_"); @@ -8417,6 +8582,48 @@ typedef struct { size_t visible_len; } visible_live_state; +/* Response state is a bounded local index over opaque response ids. When the + * optional KV cache is enabled, each record is serialized with its exact + * checkpoint and can be rebuilt after a restart; without that checkpoint it + * remains a live-process continuation only. It never copies visible history, + * so a continuation hit stays proportional to the supplied tail. */ +struct response_state_entry { + char *id; + char *checkpoint_key; + int slot_id; + int live_tokens; + /* Prompt frontier before this response generated its model output. */ + int prompt_prefix_tokens; + uint64_t frontier_epoch; + server_model_syntax model_syntax; + ds4_think_mode think_mode; + bool has_tools; + bool tool_choice_none; + bool checkpoint_saved; + char *model; + char *instructions; + char *request_tools; + char *active_tools; + tool_schema_orders tool_orders; + stop_list call_ids; + uint64_t fingerprint; + size_t bytes; + int refs; + bool indexed; + response_state_entry *prev; + response_state_entry *next; +}; + +typedef struct { + rax *by_id; + response_state_entry *head; + response_state_entry *tail; + int entries; + int max_entries; + size_t bytes; + size_t max_bytes; +} response_state_index; + struct server_slot { server *srv; int id; @@ -8425,6 +8632,14 @@ struct server_slot { live_tool_state anthropic_live; visible_live_state thinking_live; int continued_last_store_tokens; + /* Responses checkpoints retain the exact post-turn frontier, so their + * schedule tracks the continued-cache boundary they crossed rather than + * the final response token count. */ + int response_state_last_checkpoint_tokens; + /* Bumped when a request synchronizes a new prompt. A response-state + * record may use a resident session only when both frontier and epoch + * still identify the exact sampled prefix. */ + uint64_t frontier_epoch; job *assigned; job *running; @@ -8453,6 +8668,7 @@ struct server { int default_tokens; kv_disk_cache kv; tool_memory tool_mem; + response_state_index response_states; bool disable_exact_dsml_tool_replay; bool enable_cors; pthread_mutex_t tool_mu; @@ -8714,6 +8930,452 @@ static void tool_memory_free(tool_memory *m) { * KV frontier. If it does not match, DS4 falls back to the same prefix and * disk-cache machinery used by chat/completions, or returns a clear error for * tool-result-only requests that have no replayable prefix. */ +/* ========================================================================= + * Responses previous_response_id state. + * ========================================================================= + * + * This index is intentionally append-only/FIFO rather than an unbounded + * conversation store. A record owns only continuation metadata and an exact + * KV frontier reference; it never retains a second copy of the visible chat + * history. Requests pin a record while queued so eviction cannot turn a + * validated continuation into a dangling pointer. */ + +#define DS4_RESPONSE_STATE_DEFAULT_MAX_IDS 4096 +#define DS4_RESPONSE_STATE_MAX_BYTES (64u * 1024u * 1024u) + +static int response_state_max_entries(const response_state_index *idx) { + return idx && idx->max_entries > 0 ? idx->max_entries : + DS4_RESPONSE_STATE_DEFAULT_MAX_IDS; +} + +static size_t response_state_max_bytes(const response_state_index *idx) { + return idx && idx->max_bytes > 0 ? idx->max_bytes : + DS4_RESPONSE_STATE_MAX_BYTES; +} + +static bool response_state_string_eq(const char *a, const char *b) { + return !strcmp(a ? a : "", b ? b : ""); +} + +static uint64_t response_state_hash_bytes(uint64_t h, const void *data, size_t len) { + const unsigned char *p = data; + for (size_t i = 0; i < len; i++) { + h ^= p[i]; + h *= UINT64_C(1099511628211); + } + return h; +} + +static uint64_t response_state_hash_string(uint64_t h, const char *s) { + const char *text = s ? s : ""; + h = response_state_hash_bytes(h, text, strlen(text)); + const unsigned char zero = 0; + return response_state_hash_bytes(h, &zero, 1); +} + +static uint64_t response_state_fingerprint_values(server_model_syntax syntax, + ds4_think_mode think_mode, + bool has_tools, + bool tool_choice_none, + const char *model, + const char *instructions, + const char *request_tools, + const char *active_tools) { + uint64_t h = UINT64_C(1469598103934665603); + h = response_state_hash_bytes(h, &syntax, sizeof(syntax)); + h = response_state_hash_bytes(h, &think_mode, sizeof(think_mode)); + h = response_state_hash_bytes(h, &has_tools, sizeof(has_tools)); + h = response_state_hash_bytes(h, &tool_choice_none, sizeof(tool_choice_none)); + h = response_state_hash_string(h, model); + h = response_state_hash_string(h, instructions); + h = response_state_hash_string(h, request_tools); + return response_state_hash_string(h, active_tools); +} + +static size_t response_state_size_add(size_t total, size_t add) { + return add > SIZE_MAX - total ? SIZE_MAX : total + add; +} + +static size_t response_state_tool_orders_bytes(const tool_schema_orders *orders) { + size_t bytes = 0; + if (!orders) return 0; + if (orders->cap > 0) { + bytes = response_state_size_add(bytes, + (size_t)orders->cap * sizeof(orders->v[0])); + } + for (int i = 0; i < orders->len; i++) { + const tool_schema_order *o = &orders->v[i]; + if (o->cap > 0) { + bytes = response_state_size_add(bytes, + (size_t)o->cap * sizeof(o->prop[0])); + } + bytes = response_state_size_add(bytes, strlen(o->name ? o->name : "") + 1); + bytes = response_state_size_add(bytes, strlen(o->wire_name ? o->wire_name : "") + 1); + bytes = response_state_size_add(bytes, strlen(o->namespace ? o->namespace : "") + 1); + for (int j = 0; j < o->len; j++) { + bytes = response_state_size_add(bytes, strlen(o->prop[j] ? o->prop[j] : "") + 1); + } + } + return bytes; +} + +static size_t response_state_entry_bytes(const response_state_entry *entry) { + if (!entry) return 0; + size_t bytes = sizeof(*entry); + bytes = response_state_size_add(bytes, strlen(entry->id ? entry->id : "") + 1); + bytes = response_state_size_add(bytes, strlen(entry->checkpoint_key ? entry->checkpoint_key : "") + 1); + bytes = response_state_size_add(bytes, strlen(entry->model ? entry->model : "") + 1); + bytes = response_state_size_add(bytes, strlen(entry->instructions ? entry->instructions : "") + 1); + bytes = response_state_size_add(bytes, strlen(entry->request_tools ? entry->request_tools : "") + 1); + bytes = response_state_size_add(bytes, strlen(entry->active_tools ? entry->active_tools : "") + 1); + bytes = response_state_size_add(bytes, response_state_tool_orders_bytes(&entry->tool_orders)); + if (entry->call_ids.cap > 0) { + bytes = response_state_size_add(bytes, + (size_t)entry->call_ids.cap * sizeof(entry->call_ids.v[0])); + } + for (int i = 0; i < entry->call_ids.len; i++) { + bytes = response_state_size_add(bytes, + strlen(entry->call_ids.v[i] ? entry->call_ids.v[i] : "") + 1); + } + return bytes; +} + +static void response_state_entry_free(response_state_entry *entry) { + if (!entry) return; + free(entry->id); + free(entry->checkpoint_key); + free(entry->model); + free(entry->instructions); + free(entry->request_tools); + free(entry->active_tools); + tool_schema_orders_free(&entry->tool_orders); + id_list_free(&entry->call_ids); + free(entry); +} + +static void response_state_index_init_locked(response_state_index *idx) { + if (!idx || idx->by_id) return; + idx->by_id = raxNew(); + if (!idx->by_id) die("out of memory"); +} + +static void response_state_link_head(response_state_index *idx, + response_state_entry *entry) { + entry->prev = NULL; + entry->next = idx->head; + if (idx->head) idx->head->prev = entry; + else idx->tail = entry; + idx->head = entry; +} + +static void response_state_unlink(response_state_index *idx, + response_state_entry *entry) { + if (entry->prev) entry->prev->next = entry->next; + else idx->head = entry->next; + if (entry->next) entry->next->prev = entry->prev; + else idx->tail = entry->prev; + entry->prev = entry->next = NULL; +} + +static void response_state_detach_locked(response_state_index *idx, + response_state_entry *entry) { + if (!idx || !entry || !entry->indexed) return; + void *old = NULL; + (void)raxRemove(idx->by_id, (unsigned char *)entry->id, + strlen(entry->id), &old); + response_state_unlink(idx, entry); + entry->indexed = false; + if (idx->entries > 0) idx->entries--; + if (idx->bytes >= entry->bytes) idx->bytes -= entry->bytes; + else idx->bytes = 0; + if (entry->refs == 0) response_state_entry_free(entry); +} + +static void response_state_prune_locked(response_state_index *idx) { + while (idx && idx->tail && + (idx->entries > response_state_max_entries(idx) || + idx->bytes > response_state_max_bytes(idx))) + { + response_state_detach_locked(idx, idx->tail); + } +} + +static bool response_state_insert(server *s, response_state_entry *entry) { + if (!s || !entry || !entry->id || !entry->id[0]) return false; + pthread_mutex_lock(&s->tool_mu); + response_state_index *idx = &s->response_states; + response_state_index_init_locked(idx); + void *existing = raxFind(idx->by_id, (unsigned char *)entry->id, strlen(entry->id)); + if (existing != raxNotFound) { + pthread_mutex_unlock(&s->tool_mu); + return false; + } + entry->bytes = response_state_entry_bytes(entry); + if (response_state_max_entries(idx) < 1 || entry->bytes > response_state_max_bytes(idx)) { + pthread_mutex_unlock(&s->tool_mu); + return false; + } + if (!raxInsert(idx->by_id, (unsigned char *)entry->id, strlen(entry->id), entry, NULL)) { + pthread_mutex_unlock(&s->tool_mu); + die("out of memory"); + } + entry->indexed = true; + response_state_link_head(idx, entry); + idx->entries++; + idx->bytes += entry->bytes; + response_state_prune_locked(idx); + bool kept = entry->indexed; + pthread_mutex_unlock(&s->tool_mu); + return kept; +} + +static response_state_entry *response_state_acquire(server *s, const char *id) { + if (!s || !id || !id[0]) return NULL; + pthread_mutex_lock(&s->tool_mu); + response_state_entry *entry = NULL; + if (s->response_states.by_id) { + void *found = raxFind(s->response_states.by_id, (unsigned char *)id, strlen(id)); + if (found != raxNotFound) { + entry = found; + entry->refs++; + } + } + pthread_mutex_unlock(&s->tool_mu); + return entry; +} + +static bool response_state_is_current(server *s, const response_state_entry *entry, + const server_slot *slot, int live_tokens) { + if (!s || !entry || !slot) return false; + pthread_mutex_lock(&s->tool_mu); + bool ok = entry->indexed && entry->slot_id == slot->id && + entry->live_tokens == live_tokens && + entry->frontier_epoch == slot->frontier_epoch; + pthread_mutex_unlock(&s->tool_mu); + return ok; +} + +static uint64_t response_state_slot_epoch(server *s, const server_slot *slot) { + if (!s || !slot) return 0; + pthread_mutex_lock(&s->tool_mu); + uint64_t epoch = slot->frontier_epoch; + pthread_mutex_unlock(&s->tool_mu); + return epoch; +} + +static void response_state_advance_slot_epoch(server *s, server_slot *slot) { + if (!s || !slot) return; + pthread_mutex_lock(&s->tool_mu); + if (++slot->frontier_epoch == 0) slot->frontier_epoch = 1; + pthread_mutex_unlock(&s->tool_mu); +} + +static bool response_state_checkpoint_saved(server *s, + const response_state_entry *entry) { + if (!s || !entry) return false; + pthread_mutex_lock(&s->tool_mu); + bool saved = entry->checkpoint_saved; + pthread_mutex_unlock(&s->tool_mu); + return saved; +} + +static void response_state_release(server *s, response_state_entry *entry) { + if (!entry) return; + if (!s) { + /* A request can only own an entry from a live server, but keep failure + * cleanup safe for parser/unit-test construction paths. */ + return; + } + pthread_mutex_lock(&s->tool_mu); + if (entry->refs > 0) entry->refs--; + response_state_prune_locked(&s->response_states); + if (!entry->indexed && entry->refs == 0) response_state_entry_free(entry); + pthread_mutex_unlock(&s->tool_mu); +} + +static void response_state_forget(server *s, response_state_entry *entry) { + if (!s || !entry) return; + pthread_mutex_lock(&s->tool_mu); + response_state_detach_locked(&s->response_states, entry); + pthread_mutex_unlock(&s->tool_mu); +} + +static void response_state_index_free(response_state_index *idx) { + if (!idx) return; + while (idx->tail) response_state_detach_locked(idx, idx->tail); + if (idx->by_id) raxFree(idx->by_id); + memset(idx, 0, sizeof(*idx)); +} + +static response_state_entry *response_state_make(const char *id, int slot_id, + int live_tokens, uint64_t frontier_epoch, + const request *r, + const tool_calls *calls) { + if (!id || !id[0] || !r) return NULL; + response_state_entry *entry = xmalloc(sizeof(*entry)); + memset(entry, 0, sizeof(*entry)); + entry->id = xstrdup(id); + size_t key_len = strlen("responses-id:") + strlen(id) + 1; + entry->checkpoint_key = xmalloc(key_len); + snprintf(entry->checkpoint_key, key_len, "responses-id:%s", id); + entry->slot_id = slot_id; + entry->live_tokens = live_tokens; + entry->prompt_prefix_tokens = r->cache_read_tokens + r->cache_write_tokens; + if (entry->prompt_prefix_tokens < 0 || entry->prompt_prefix_tokens > live_tokens) { + entry->prompt_prefix_tokens = live_tokens; + } + entry->frontier_epoch = frontier_epoch; + entry->model_syntax = r->model_syntax; + entry->think_mode = r->think_mode; + entry->has_tools = r->has_tools; + entry->tool_choice_none = r->responses_state_tool_choice_none; + entry->model = xstrdup(r->model ? r->model : ""); + entry->instructions = xstrdup(r->responses_state_instructions ? + r->responses_state_instructions : ""); + entry->request_tools = xstrdup(r->responses_state_request_tools ? + r->responses_state_request_tools : ""); + entry->active_tools = xstrdup(r->responses_state_active_tools ? + r->responses_state_active_tools : ""); + tool_schema_orders_clone_append(&entry->tool_orders, &r->tool_orders); + if (calls) { + for (int i = 0; i < calls->len; i++) id_list_push_unique(&entry->call_ids, calls->v[i].id); + } + entry->fingerprint = response_state_fingerprint_values( + entry->model_syntax, entry->think_mode, entry->has_tools, + entry->tool_choice_none, entry->model, entry->instructions, + entry->request_tools, entry->active_tools); + return entry; +} + +static response_state_entry *response_state_remember(server *s, int slot_id, + int live_tokens, uint64_t frontier_epoch, + const char *id, const request *r, + const tool_calls *calls) { + response_state_entry *entry = response_state_make(id, slot_id, live_tokens, + frontier_epoch, r, calls); + if (!entry) return NULL; + /* Keep a private reference while the producer optionally writes its disk + * checkpoint; index eviction may otherwise reclaim a just-created record. */ + entry->refs = 1; + if (response_state_insert(s, entry)) return entry; + entry->refs = 0; + response_state_entry_free(entry); + return NULL; +} + +static void response_state_set_checkpoint_saved(server *s, response_state_entry *entry, + bool checkpoint_saved) { + if (!s || !entry) return; + pthread_mutex_lock(&s->tool_mu); + if (entry->indexed) entry->checkpoint_saved = checkpoint_saved; + pthread_mutex_unlock(&s->tool_mu); +} + +static void responses_new_response_id(server *s, char *dst, size_t dstlen) { + if (!dst || dstlen == 0) return; + for (int attempt = 0; attempt < 16; attempt++) { + responses_random_id(dst, dstlen, "resp_"); + response_state_entry *existing = response_state_acquire(s, dst); + if (!existing) return; + response_state_release(s, existing); + } + /* A 96-bit random collision is already astronomically unlikely; the final + * draw remains a usable opaque id if an adversarial test exhausts retries. */ + responses_random_id(dst, dstlen, "resp_"); +} + +static const stop_list *response_state_call_ids(const response_state_entry *entry) { + return entry ? &entry->call_ids : NULL; +} + +static bool response_state_config_matches(const response_state_entry *entry, + const request *r) { + if (!entry || !r) return false; + if (entry->model_syntax != r->model_syntax) return false; + if (r->model_from_request && !response_state_string_eq(entry->model, r->model)) return false; + if (r->responses_state_instructions_set && + !response_state_string_eq(entry->instructions, r->responses_state_instructions)) return false; + if (r->responses_state_request_tools_set && + !response_state_string_eq(entry->request_tools, r->responses_state_request_tools)) return false; + if (r->responses_state_tool_choice_set && + entry->tool_choice_none != r->responses_state_tool_choice_none) return false; + if (r->responses_state_thinking_set && entry->think_mode != r->think_mode) return false; + return true; +} + +static bool responses_input_is_append_only(const chat_msgs *msgs) { + if (!msgs) return false; + for (int i = 0; i < msgs->len; i++) { + const char *role = msgs->v[i].role ? msgs->v[i].role : ""; + if (strcmp(role, "user") && strcmp(role, "tool") && strcmp(role, "function")) return false; + } + return true; +} + +static bool responses_validate_state_tool_outputs(const chat_msgs *msgs, + const stop_list *known_ids, + char *err, size_t errlen) { + if (!msgs) return true; + for (int i = 0; i < msgs->len; i++) { + const chat_msg *m = &msgs->v[i]; + if (strcmp(m->role ? m->role : "", "tool") && + strcmp(m->role ? m->role : "", "function")) continue; + stop_list ids = {0}; + chat_msg_collect_tool_call_ids(m, &ids); + if (ids.len == 0) { + snprintf(err, errlen, + "previous_response_id tool output requires a call_id; replay the full input history for an edited branch"); + id_list_free(&ids); + return false; + } + for (int j = 0; j < ids.len; j++) { + if (!id_list_contains(known_ids, ids.v[j])) { + snprintf(err, errlen, + "previous_response_id does not contain call_id %s; replay the full input history for an edited branch", + ids.v[j] ? ids.v[j] : ""); + id_list_free(&ids); + return false; + } + } + id_list_free(&ids); + } + return true; +} + +static void response_state_apply(request *r, response_state_entry *entry, + const char *delta_tools) { + if (!r || !entry) return; + tool_schema_orders merged = {0}; + tool_schema_orders_clone_append(&merged, &entry->tool_orders); + tool_schema_orders_clone_append(&merged, &r->tool_orders); + tool_schema_orders_free(&r->tool_orders); + r->tool_orders = merged; + + free(r->model); + r->model = xstrdup(entry->model); + /* The inherited model is now explicit for the rest of the server pipeline; + * client_main must not replace it with today's default alias. */ + r->model_from_request = true; + r->model_syntax = entry->model_syntax; + r->think_mode = entry->think_mode; + + buf active = {0}; + if (entry->active_tools && entry->active_tools[0]) buf_puts(&active, entry->active_tools); + if (!entry->tool_choice_none && delta_tools && delta_tools[0]) { + if (active.len) buf_putc(&active, '\n'); + buf_puts(&active, delta_tools); + } + r->has_tools = !entry->tool_choice_none && active.len > 0; + responses_request_set_state_config(r, entry->instructions, entry->request_tools, + active.ptr ? active.ptr : "", entry->tool_choice_none); + r->responses_state_fingerprint = response_state_fingerprint_values( + r->model_syntax, r->think_mode, r->has_tools, + r->responses_state_tool_choice_none, r->model, + r->responses_state_instructions, r->responses_state_request_tools, + r->responses_state_active_tools); + buf_free(&active); +} + static void live_tool_state_clear_locked(live_tool_state *st) { if (!st) return; stop_list_clear(&st->call_ids); @@ -9062,11 +9724,23 @@ static void apply_anthropic_stream_tool_ids(tool_calls *calls, #define KV_EXT_TOOL_MAP DS4_KVSTORE_EXT_TOOL_MAP #define KV_EXT_RESPONSES_VISIBLE DS4_KVSTORE_EXT_RESPONSES_VISIBLE #define KV_EXT_THINKING_VISIBLE DS4_KVSTORE_EXT_THINKING_VISIBLE +#define KV_EXT_RESPONSES_ID DS4_KVSTORE_EXT_RESPONSES_ID #define KV_TOOL_MAP_MAGIC0 'K' #define KV_TOOL_MAP_MAGIC1 'T' #define KV_TOOL_MAP_MAGIC2 'M' #define KV_TOOL_MAP_VERSION 1u #define KV_TOOL_MAP_HEADER 8u +/* Response-id records live in the checkpoint trailer, before the optional + * tool map. The metadata is intentionally separate from the cache key: the + * key stays the opaque response id while the trailer carries the configuration + * and bindings needed to validate a continuation after a server restart. */ +#define KV_RESPONSE_STATE_MAGIC0 'K' +#define KV_RESPONSE_STATE_MAGIC1 'R' +#define KV_RESPONSE_STATE_MAGIC2 'S' +#define KV_RESPONSE_STATE_VERSION 1u +#define KV_RESPONSE_STATE_HEADER 8u +#define KV_RESPONSE_STATE_MAX_ITEMS (DS4_RESPONSE_STATE_DEFAULT_MAX_IDS * 4u) +#define KV_RESPONSE_STATE_MAX_ID_BYTES 256u typedef enum { KV_REASON_UNKNOWN = DS4_KVSTORE_REASON_UNKNOWN, @@ -9310,6 +9984,354 @@ static int kv_tool_map_load_from_pos(server *s, FILE *fp, const stop_list *wante return loaded; } +/* The response-id cache key identifies the checkpoint; this bounded trailer + * identifies how that checkpoint may be resumed. Keep it binary rather than + * reparsing a JSON response on restart, and reject malformed or oversized + * records before they can enter the in-memory index. */ +static bool kv_response_state_size_add(uint64_t *total, uint64_t add) { + if (!total || add > UINT64_MAX - *total) return false; + *total += add; + return true; +} + +static bool kv_response_state_size_string(uint64_t *total, const char *text) { + size_t len = strlen(text ? text : ""); + if (len > UINT32_MAX) return false; + return kv_response_state_size_add(total, 4u) && + kv_response_state_size_add(total, (uint64_t)len); +} + +static bool kv_response_state_section_size(const response_state_entry *entry, + uint64_t *bytes_out) { + if (bytes_out) *bytes_out = 0; + if (!entry || !entry->id || !entry->id[0] || !entry->checkpoint_key || + !entry->checkpoint_key[0] || entry->live_tokens <= 0 || + entry->prompt_prefix_tokens < 0 || + entry->prompt_prefix_tokens > entry->live_tokens || + entry->tool_orders.len < 0 || entry->call_ids.len < 0 || + (uint64_t)entry->tool_orders.len > KV_RESPONSE_STATE_MAX_ITEMS || + (uint64_t)entry->call_ids.len > KV_RESPONSE_STATE_MAX_ITEMS) + { + return false; + } + + /* Five u32 fields, a u64 fingerprint, six strings, then the two lists. */ + uint64_t bytes = KV_RESPONSE_STATE_HEADER + 28u; + const char *strings[] = { + entry->id, entry->checkpoint_key, entry->model, entry->instructions, + entry->request_tools, entry->active_tools, + }; + for (size_t i = 0; i < sizeof(strings) / sizeof(strings[0]); i++) { + if (!kv_response_state_size_string(&bytes, strings[i])) return false; + } + if (!kv_response_state_size_add(&bytes, 4u)) return false; + for (int i = 0; i < entry->tool_orders.len; i++) { + const tool_schema_order *order = &entry->tool_orders.v[i]; + if (order->len < 0 || (uint64_t)order->len > KV_RESPONSE_STATE_MAX_ITEMS || + !kv_response_state_size_add(&bytes, 8u) || + !kv_response_state_size_string(&bytes, order->name) || + !kv_response_state_size_string(&bytes, order->wire_name) || + !kv_response_state_size_string(&bytes, order->namespace)) + { + return false; + } + for (int j = 0; j < order->len; j++) { + if (!kv_response_state_size_string(&bytes, order->prop[j])) return false; + } + } + if (!kv_response_state_size_add(&bytes, 4u)) return false; + for (int i = 0; i < entry->call_ids.len; i++) { + if (!entry->call_ids.v[i] || !entry->call_ids.v[i][0] || + !kv_response_state_size_string(&bytes, entry->call_ids.v[i])) + { + return false; + } + } + if (bytes < KV_RESPONSE_STATE_HEADER || + bytes - KV_RESPONSE_STATE_HEADER > UINT32_MAX || + bytes > DS4_RESPONSE_STATE_MAX_BYTES) + { + return false; + } + if (bytes_out) *bytes_out = bytes; + return true; +} + +static void kv_response_state_put_u32(buf *out, uint32_t value) { + uint8_t bytes[4]; + le_put32(bytes, value); + buf_append(out, bytes, sizeof(bytes)); +} + +static void kv_response_state_put_u64(buf *out, uint64_t value) { + kv_response_state_put_u32(out, (uint32_t)value); + kv_response_state_put_u32(out, (uint32_t)(value >> 32)); +} + +static void kv_response_state_put_string(buf *out, const char *text) { + const char *value = text ? text : ""; + size_t len = strlen(value); + kv_response_state_put_u32(out, (uint32_t)len); + buf_append(out, value, len); +} + +static bool kv_response_state_build_section(const response_state_entry *entry, + buf *out) { + uint64_t expected = 0; + if (!out || !kv_response_state_section_size(entry, &expected)) return false; + + const uint8_t header[KV_RESPONSE_STATE_HEADER] = { + KV_RESPONSE_STATE_MAGIC0, KV_RESPONSE_STATE_MAGIC1, + KV_RESPONSE_STATE_MAGIC2, KV_RESPONSE_STATE_VERSION, + 0, 0, 0, 0, + }; + buf_append(out, header, sizeof(header)); + kv_response_state_put_u32(out, (uint32_t)entry->live_tokens); + kv_response_state_put_u32(out, (uint32_t)entry->prompt_prefix_tokens); + kv_response_state_put_u32(out, (uint32_t)entry->model_syntax); + kv_response_state_put_u32(out, (uint32_t)entry->think_mode); + uint32_t flags = (entry->has_tools ? 1u : 0u) | + (entry->tool_choice_none ? 2u : 0u); + kv_response_state_put_u32(out, flags); + kv_response_state_put_u64(out, entry->fingerprint); + kv_response_state_put_string(out, entry->id); + kv_response_state_put_string(out, entry->checkpoint_key); + kv_response_state_put_string(out, entry->model); + kv_response_state_put_string(out, entry->instructions); + kv_response_state_put_string(out, entry->request_tools); + kv_response_state_put_string(out, entry->active_tools); + kv_response_state_put_u32(out, (uint32_t)entry->tool_orders.len); + for (int i = 0; i < entry->tool_orders.len; i++) { + const tool_schema_order *order = &entry->tool_orders.v[i]; + kv_response_state_put_u32(out, order->responses_tool_search ? 1u : 0u); + kv_response_state_put_string(out, order->name); + kv_response_state_put_string(out, order->wire_name); + kv_response_state_put_string(out, order->namespace); + kv_response_state_put_u32(out, (uint32_t)order->len); + for (int j = 0; j < order->len; j++) { + kv_response_state_put_string(out, order->prop[j]); + } + } + kv_response_state_put_u32(out, (uint32_t)entry->call_ids.len); + for (int i = 0; i < entry->call_ids.len; i++) { + kv_response_state_put_string(out, entry->call_ids.v[i]); + } + + if (out->len != expected || out->len < KV_RESPONSE_STATE_HEADER) { + buf_free(out); + return false; + } + le_put32((uint8_t *)out->ptr + 4, + (uint32_t)(out->len - KV_RESPONSE_STATE_HEADER)); + return true; +} + +typedef struct { + const uint8_t *ptr; + size_t len; + size_t pos; +} kv_response_state_reader; + +static bool kv_response_state_read_u32(kv_response_state_reader *reader, + uint32_t *out) { + if (!reader || !out || reader->pos > reader->len || + reader->len - reader->pos < 4u) return false; + *out = le_get32(reader->ptr + reader->pos); + reader->pos += 4u; + return true; +} + +static bool kv_response_state_read_u64(kv_response_state_reader *reader, + uint64_t *out) { + uint32_t lo = 0, hi = 0; + if (!kv_response_state_read_u32(reader, &lo) || + !kv_response_state_read_u32(reader, &hi)) return false; + *out = (uint64_t)lo | ((uint64_t)hi << 32); + return true; +} + +static bool kv_response_state_read_string(kv_response_state_reader *reader, + char **out) { + if (out) *out = NULL; + uint32_t len = 0; + if (!reader || !out || !kv_response_state_read_u32(reader, &len) || + len > DS4_RESPONSE_STATE_MAX_BYTES || reader->pos > reader->len || + (size_t)len > reader->len - reader->pos || + memchr(reader->ptr + reader->pos, '\0', len) != NULL) + { + return false; + } + *out = xstrndup((const char *)reader->ptr + reader->pos, len); + reader->pos += len; + return true; +} + +static bool kv_response_state_read_header(FILE *fp, uint32_t *payload_bytes) { + uint8_t header[KV_RESPONSE_STATE_HEADER]; + if (!fp || !payload_bytes || fread(header, 1, sizeof(header), fp) != sizeof(header)) { + return false; + } + if (header[0] != KV_RESPONSE_STATE_MAGIC0 || + header[1] != KV_RESPONSE_STATE_MAGIC1 || + header[2] != KV_RESPONSE_STATE_MAGIC2 || + header[3] != KV_RESPONSE_STATE_VERSION) + { + return false; + } + *payload_bytes = le_get32(header + 4); + return *payload_bytes <= DS4_RESPONSE_STATE_MAX_BYTES; +} + +static bool kv_response_state_skip_from_pos(FILE *fp) { + uint32_t payload_bytes = 0; + return kv_response_state_read_header(fp, &payload_bytes) && + fseeko(fp, (off_t)payload_bytes, SEEK_CUR) == 0; +} + +/* A generic cache load normally starts at a tool-map trailer. Response-id + * files add their state section first, so detect and skip it without making + * ordinary checkpoints depend on Responses state. */ +static bool kv_response_state_maybe_skip_from_pos(FILE *fp) { + if (!fp) return false; + off_t pos = ftello(fp); + if (pos < 0) return false; + uint8_t magic[4]; + size_t got = fread(magic, 1, sizeof(magic), fp); + bool at_eof = got == 0 && feof(fp); + if (fseeko(fp, pos, SEEK_SET) != 0) return false; + if (at_eof) { + clearerr(fp); + return true; + } + if (got != sizeof(magic)) return false; + if (magic[0] != KV_RESPONSE_STATE_MAGIC0 || + magic[1] != KV_RESPONSE_STATE_MAGIC1 || + magic[2] != KV_RESPONSE_STATE_MAGIC2 || + magic[3] != KV_RESPONSE_STATE_VERSION) + { + return true; + } + return kv_response_state_skip_from_pos(fp); +} + +static response_state_entry *kv_response_state_read_from_pos(FILE *fp, + const char *wanted_id, + uint32_t expected_tokens) { + uint32_t payload_bytes = 0; + if (!fp || !wanted_id || !wanted_id[0] || + !kv_response_state_read_header(fp, &payload_bytes) || payload_bytes == 0) + { + return NULL; + } + uint8_t *payload = xmalloc(payload_bytes); + if (fread(payload, 1, payload_bytes, fp) != payload_bytes) { + free(payload); + return NULL; + } + + response_state_entry *entry = xmalloc(sizeof(*entry)); + memset(entry, 0, sizeof(*entry)); + kv_response_state_reader reader = {.ptr = payload, .len = payload_bytes}; + uint32_t live_tokens = 0, prompt_prefix_tokens = 0; + uint32_t syntax = 0, think_mode = 0, flags = 0, order_count = 0, call_count = 0; + bool ok = kv_response_state_read_u32(&reader, &live_tokens) && + kv_response_state_read_u32(&reader, &prompt_prefix_tokens) && + kv_response_state_read_u32(&reader, &syntax) && + kv_response_state_read_u32(&reader, &think_mode) && + kv_response_state_read_u32(&reader, &flags) && + kv_response_state_read_u64(&reader, &entry->fingerprint) && + kv_response_state_read_string(&reader, &entry->id) && + kv_response_state_read_string(&reader, &entry->checkpoint_key) && + kv_response_state_read_string(&reader, &entry->model) && + kv_response_state_read_string(&reader, &entry->instructions) && + kv_response_state_read_string(&reader, &entry->request_tools) && + kv_response_state_read_string(&reader, &entry->active_tools) && + kv_response_state_read_u32(&reader, &order_count); + if (!ok || live_tokens == 0 || live_tokens > INT_MAX || + prompt_prefix_tokens > live_tokens || syntax > SERVER_MODEL_SYNTAX_GLM || + think_mode > DS4_THINK_MAX || flags > 3u || + order_count > KV_RESPONSE_STATE_MAX_ITEMS) + { + goto bad; + } + entry->live_tokens = (int)live_tokens; + entry->prompt_prefix_tokens = (int)prompt_prefix_tokens; + entry->model_syntax = (server_model_syntax)syntax; + entry->think_mode = (ds4_think_mode)think_mode; + entry->has_tools = (flags & 1u) != 0; + entry->tool_choice_none = (flags & 2u) != 0; + + for (uint32_t i = 0; i < order_count; i++) { + tool_schema_order order = {0}; + uint32_t order_flags = 0, prop_count = 0; + ok = kv_response_state_read_u32(&reader, &order_flags) && + kv_response_state_read_string(&reader, &order.name) && + kv_response_state_read_string(&reader, &order.wire_name) && + kv_response_state_read_string(&reader, &order.namespace) && + kv_response_state_read_u32(&reader, &prop_count); + if (!ok || order_flags > 1u || prop_count > KV_RESPONSE_STATE_MAX_ITEMS || + !order.name || !order.name[0] || + tool_schema_orders_find(&entry->tool_orders, order.name)) + { + tool_schema_order_free(&order); + goto bad; + } + order.responses_tool_search = order_flags != 0; + for (uint32_t j = 0; j < prop_count; j++) { + char *prop = NULL; + if (!kv_response_state_read_string(&reader, &prop)) { + tool_schema_order_free(&order); + goto bad; + } + tool_schema_order_prop_push(&order, prop); + } + tool_schema_orders_push(&entry->tool_orders, order); + } + + if (!kv_response_state_read_u32(&reader, &call_count) || + call_count > KV_RESPONSE_STATE_MAX_ITEMS) + { + goto bad; + } + for (uint32_t i = 0; i < call_count; i++) { + char *call_id = NULL; + if (!kv_response_state_read_string(&reader, &call_id) || !call_id[0] || + id_list_contains(&entry->call_ids, call_id)) + { + free(call_id); + goto bad; + } + stop_list_push(&entry->call_ids, call_id); + } + + size_t wanted_len = strlen(wanted_id); + size_t key_len = strlen("responses-id:") + wanted_len + 1; + char *wanted_key = xmalloc(key_len); + snprintf(wanted_key, key_len, "responses-id:%s", wanted_id); + uint64_t fingerprint = response_state_fingerprint_values( + entry->model_syntax, entry->think_mode, entry->has_tools, + entry->tool_choice_none, entry->model, entry->instructions, + entry->request_tools, entry->active_tools); + ok = reader.pos == reader.len && !strcmp(entry->id, wanted_id) && + !strcmp(entry->checkpoint_key, wanted_key) && + entry->live_tokens == (int)expected_tokens && + entry->fingerprint == fingerprint; + free(wanted_key); + if (!ok) goto bad; + + entry->slot_id = -1; + entry->checkpoint_saved = true; + entry->bytes = response_state_entry_bytes(entry); + if (entry->bytes > response_state_max_bytes(NULL)) goto bad; + free(payload); + return entry; + +bad: + free(payload); + response_state_entry_free(entry); + return NULL; +} + #ifdef DS4_SERVER_TEST static void kv_fill_header(uint8_t h[KV_CACHE_FIXED_HEADER], uint8_t quant_bits, uint8_t reason, uint8_t ext_flags, @@ -9361,7 +10383,11 @@ static void kv_cache_restore_tool_memory_for_messages(server *s, const chat_msgs skip <= (uint64_t)INT64_MAX && fseeko(fp, (off_t)skip, SEEK_CUR) == 0) { - kv_tool_map_load_from_pos(s, fp, &wanted); + bool trailer_ok = true; + if (hdr.ext_flags & KV_EXT_RESPONSES_ID) { + trailer_ok = kv_response_state_skip_from_pos(fp); + } + if (trailer_ok) kv_tool_map_load_from_pos(s, fp, &wanted); } fclose(fp); } @@ -9428,6 +10454,15 @@ static void build_prompt_from_exact_prefix_and_text_suffix( engine, exact_prefix, suffix_text, out); } +static void build_prompt_from_exact_prefix_and_token_suffix( + const ds4_tokens *exact_prefix, + const ds4_tokens *suffix, + ds4_tokens *out) +{ + ds4_tokens_copy(out, exact_prefix); + for (int i = 0; suffix && i < suffix->len; i++) ds4_tokens_push(out, suffix->v[i]); +} + static int kv_cache_store_len(const kv_disk_cache *kc, int tokens) { return ds4_kvstore_store_len(kc, tokens); } @@ -9451,7 +10486,6 @@ static int kv_cache_continued_store_target(const kv_disk_cache *kc, int live_tok -#ifdef DS4_SERVER_TEST static bool kv_cache_file_size_fits(const kv_disk_cache *kc, uint64_t text_bytes, uint64_t payload_bytes, @@ -9462,7 +10496,6 @@ static bool kv_cache_file_size_fits(const kv_disk_cache *kc, tool_map_bytes, file_bytes_out, required_bytes_out); } -#endif @@ -9476,10 +10509,80 @@ static bool kv_cache_tool_map_write_cb(void *ud, FILE *fp, const char *text, return kv_tool_map_write((server *)ud, fp, text, written_bytes); } +typedef struct { + server *server; + const response_state_entry *entry; +} kv_response_state_trailer; + +static bool kv_response_state_trailer_size_cb(void *ud, const char *text, + uint64_t *bytes_out) { + if (bytes_out) *bytes_out = 0; + kv_response_state_trailer *trailer = ud; + uint64_t state_bytes = 0, tool_map_bytes = 0, total = 0; + if (!trailer || !trailer->server || !trailer->entry || + !kv_response_state_section_size(trailer->entry, &state_bytes) || + !kv_tool_map_serialized_size(trailer->server, text, &tool_map_bytes)) + { + return false; + } + /* A zero-count map makes the trailer order unambiguous and lets normal KV + * loaders continue to restore any exact DSML mappings after the state part. */ + if (tool_map_bytes == 0) tool_map_bytes = KV_TOOL_MAP_HEADER; + if (!kv_response_state_size_add(&total, state_bytes) || + !kv_response_state_size_add(&total, tool_map_bytes)) + { + return false; + } + if (bytes_out) *bytes_out = total; + return true; +} + +static bool kv_response_state_write_empty_tool_map(FILE *fp) { + uint8_t header[KV_TOOL_MAP_HEADER] = { + KV_TOOL_MAP_MAGIC0, KV_TOOL_MAP_MAGIC1, KV_TOOL_MAP_MAGIC2, + KV_TOOL_MAP_VERSION, 0, 0, 0, 0, + }; + return fwrite(header, 1, sizeof(header), fp) == sizeof(header); +} + +static bool kv_response_state_trailer_write_cb(void *ud, FILE *fp, + const char *text, + uint64_t *written_bytes) { + if (written_bytes) *written_bytes = 0; + kv_response_state_trailer *trailer = ud; + if (!trailer || !trailer->server || !trailer->entry || !fp) return false; + + buf section = {0}; + if (!kv_response_state_build_section(trailer->entry, §ion)) return false; + bool ok = fwrite(section.ptr, 1, section.len, fp) == section.len; + uint64_t tool_map_bytes = 0; + if (ok) ok = kv_tool_map_write(trailer->server, fp, text, &tool_map_bytes); + if (ok && tool_map_bytes == 0) { + ok = kv_response_state_write_empty_tool_map(fp); + tool_map_bytes = ok ? KV_TOOL_MAP_HEADER : 0; + } + if (ok && written_bytes) *written_bytes = section.len + tool_map_bytes; + buf_free(§ion); + return ok; +} + static int kv_cache_tool_map_load_cb(void *ud, FILE *fp, const void *wanted) { + if (!kv_response_state_maybe_skip_from_pos(fp)) return 0; return kv_tool_map_load_from_pos((server *)ud, fp, (const stop_list *)wanted); } +static ds4_kvstore_trailer_hooks kv_cache_response_state_hooks( + kv_response_state_trailer *trailer, const stop_list *wanted) { + return (ds4_kvstore_trailer_hooks){ + .ud = trailer, + .ext_flag = (uint8_t)(KV_EXT_RESPONSES_ID | KV_EXT_TOOL_MAP), + .serialized_size = kv_response_state_trailer_size_cb, + .write = kv_response_state_trailer_write_cb, + .load = kv_cache_tool_map_load_cb, + .load_wanted = wanted, + }; +} + static ds4_kvstore_trailer_hooks kv_cache_tool_map_hooks(server *s, const stop_list *wanted) { return (ds4_kvstore_trailer_hooks){ @@ -9505,7 +10608,7 @@ static bool kv_cache_store_live_prefix_text(server *s, server_slot *slot, pthread_mutex_lock(&s->kv_mu); bool ok = ds4_kvstore_store_live_prefix_text(&s->kv, s->engine, slot->session, - tokens, store_len, reason, + tokens, store_len, false, reason, cache_text_override, cache_text_ext, cache_text_key, @@ -9515,6 +10618,86 @@ static bool kv_cache_store_live_prefix_text(server *s, server_slot *slot, return ok; } +/* Response-id checkpoints must carry their validation metadata in the same + * atomic KV file as the saved model frontier. A restart can therefore rebuild + * the small index without replaying visible history or trusting a sidecar. */ +static bool kv_cache_store_response_state_prefix(server *s, server_slot *slot, + const ds4_tokens *tokens, + const response_state_entry *entry) { + if (!s || !slot || !tokens || !entry || !entry->checkpoint_key) return false; + char err[160] = {0}; + kv_response_state_trailer trailer = {.server = s, .entry = entry}; + ds4_kvstore_trailer_hooks hooks = kv_cache_response_state_hooks(&trailer, NULL); + + /* Most session implementations can report the exact payload size without + * staging it. Reject a known-overbudget response checkpoint before copying + * a full KV image to the temporary staging file. Distributed sessions that + * cannot know the size yet still use the regular staged budget check. */ + uint64_t trailer_bytes = 0; + if (!kv_response_state_trailer_size_cb(&trailer, entry->checkpoint_key, + &trailer_bytes)) { + server_log(DS4_LOG_WARNING, + "ds4-server: response-id checkpoint skipped tokens=%d because its trailer size is invalid", + tokens->len); + return false; + } + + pthread_mutex_lock(&s->inference_mu); + pthread_mutex_lock(&s->kv_mu); + const uint64_t payload_bytes = ds4_session_payload_bytes(slot->session); + bool fits = true; + if (payload_bytes != 0) { + uint64_t file_bytes = 0, required_bytes = 0; + fits = kv_cache_file_size_fits(&s->kv, + (uint64_t)strlen(entry->checkpoint_key), + payload_bytes, trailer_bytes, + &file_bytes, &required_bytes); + if (!fits) { + server_log(DS4_LOG_KVCACHE, + "ds4-server: response-id checkpoint skipped tokens=%d because estimated file size %.2f MiB (%.2f MiB with safety) exceeds budget %.2f MiB before staging", + tokens->len, + (double)file_bytes / (1024.0 * 1024.0), + (double)required_bytes / (1024.0 * 1024.0), + (double)s->kv.budget_bytes / (1024.0 * 1024.0)); + } + } + bool ok = fits && ds4_kvstore_store_live_prefix_text( + &s->kv, s->engine, slot->session, tokens, tokens->len, false, "continued", + entry->checkpoint_key, KV_EXT_RESPONSES_ID, "responses-id", + &hooks, err, sizeof(err)); + pthread_mutex_unlock(&s->kv_mu); + pthread_mutex_unlock(&s->inference_mu); + return ok; +} + +/* A response id is published only after its final wire write succeeds. If the + * peer disconnects, discard a checkpoint staged for that unpublished id too. */ +static void kv_cache_discard_response_state_checkpoint( + server *s, const response_state_entry *entry) { + if (!s || !s->kv.enabled || !s->kv.dir || !entry || + !entry->checkpoint_key || !entry->checkpoint_key[0] || + !response_state_checkpoint_saved(s, entry)) + { + return; + } + char sha[41]; + ds4_kvstore_sha1_bytes_hex(entry->checkpoint_key, + strlen(entry->checkpoint_key), sha); + char *path = ds4_kvstore_path_for_sha(&s->kv, sha); + if (!path) return; + pthread_mutex_lock(&s->kv_mu); + if (unlink(path) == 0) { + server_log(DS4_LOG_KVCACHE, + "ds4-server: response-id checkpoint discarded after client disconnect"); + } else if (errno != ENOENT) { + server_log(DS4_LOG_WARNING, + "ds4-server: response-id checkpoint cleanup failed: %s", + strerror(errno)); + } + pthread_mutex_unlock(&s->kv_mu); + free(path); +} + static bool kv_cache_store_live_prefix(server *s, server_slot *slot, const ds4_tokens *tokens, int store_len, const char *reason) { @@ -9608,6 +10791,36 @@ static void kv_cache_slot_restore_suppressed(server_slot *slot, } } +/* Response ids need their exact post-turn frontier, not merely the aligned + * prefix used by ordinary continued entries. Trigger that more expensive save + * only after crossing the same continued-cache boundary, and remember the + * attempted boundary before I/O so an oversized/failed checkpoint cannot be + * retried on every short response. */ +static int response_state_checkpoint_boundary(const server *s, + const server_slot *slot, + int live_tokens) { + if (!s || !slot) return 0; + return ds4_kvstore_continued_store_crossed_target( + &s->kv, slot->response_state_last_checkpoint_tokens, live_tokens); +} + +static void response_state_note_checkpoint_attempt(server_slot *slot, + int boundary) { + if (slot && boundary > slot->response_state_last_checkpoint_tokens) + slot->response_state_last_checkpoint_tokens = boundary; +} + +static void response_state_reset_checkpoint_cadence(server_slot *slot) { + if (slot) slot->response_state_last_checkpoint_tokens = 0; +} + +static void response_state_seed_checkpoint_cadence(server *s, + server_slot *slot, + int live_tokens) { + const int boundary = response_state_checkpoint_boundary(s, slot, live_tokens); + if (boundary > 0) response_state_note_checkpoint_attempt(slot, boundary); +} + static void kv_cache_discard_failed_disk_entry(server *s, server_slot *slot, const char *path) { if (!s || !slot || !path) return; @@ -9623,6 +10836,7 @@ static void kv_cache_discard_failed_disk_entry(server *s, server_slot *slot, } pthread_mutex_unlock(&s->kv_mu); slot->continued_last_store_tokens = 0; + response_state_reset_checkpoint_cadence(slot); pthread_mutex_lock(&s->inference_mu); ds4_session_invalidate(slot->session); pthread_mutex_unlock(&s->inference_mu); @@ -9653,7 +10867,8 @@ static int kv_cache_try_load_text(server *s, server_slot *slot, ds4_tokens *effective_prompt, char **loaded_path_out, uint8_t *loaded_ext_flags_out, - bool responses_protocol) { + bool responses_protocol, + uint8_t required_ext_flags) { if (!s || !slot) return 0; if (loaded_path_out) *loaded_path_out = NULL; if (loaded_ext_flags_out) *loaded_ext_flags_out = 0; @@ -9663,7 +10878,7 @@ static int kv_cache_try_load_text(server *s, server_slot *slot, pthread_mutex_lock(&s->kv_mu); int loaded = ds4_kvstore_try_load_text(&s->kv, s->engine, slot->session, prompt_text, effective_prompt, &lr, - &hooks, responses_protocol); + &hooks, responses_protocol, required_ext_flags); pthread_mutex_unlock(&s->kv_mu); pthread_mutex_unlock(&s->inference_mu); if (loaded > 0) { @@ -9682,7 +10897,66 @@ static int kv_cache_try_load(server *s, server_slot *slot, const request *req, effective_prompt, loaded_path_out, loaded_ext_flags_out, - req && req->api == API_RESPONSES); + req && req->api == API_RESPONSES, 0); +} + +/* Recreate one bounded state record directly from its opaque checkpoint key. + * This is deliberately lazy: a restarted server does O(1) work for the id a + * client actually resumes instead of scanning every retained KV payload. */ +static response_state_entry *response_state_read_checkpoint(server *s, + const char *id) { + if (!s || !s->kv.enabled || !s->kv.dir || !id || !id[0] || + strlen(id) > KV_RESPONSE_STATE_MAX_ID_BYTES) + { + return NULL; + } + size_t key_len = strlen("responses-id:") + strlen(id) + 1; + char *key = xmalloc(key_len); + snprintf(key, key_len, "responses-id:%s", id); + char sha[41]; + ds4_kvstore_sha1_bytes_hex(key, strlen(key), sha); + char *path = ds4_kvstore_path_for_sha(&s->kv, sha); + + response_state_entry *entry = NULL; + pthread_mutex_lock(&s->kv_mu); + FILE *fp = fopen(path, "rb"); + if (fp) { + kv_entry hdr = {0}; + uint32_t text_bytes = 0; + bool valid = kv_read_header(fp, &hdr, &text_bytes) && + (hdr.ext_flags & KV_EXT_RESPONSES_ID) && + text_bytes == strlen(key) && + hdr.tokens > 0 && hdr.ctx_size <= (uint32_t)s->ctx_size && + (!s->engine || hdr.model_id == (uint8_t)ds4_engine_model_id(s->engine)); + if (valid) { + char *text = xmalloc((size_t)text_bytes + 1); + valid = fread(text, 1, text_bytes, fp) == text_bytes; + text[text_bytes] = '\0'; + valid = valid && !memcmp(text, key, text_bytes); + free(text); + } + if (valid && hdr.payload_bytes <= INT64_MAX && + fseeko(fp, (off_t)hdr.payload_bytes, SEEK_CUR) == 0) + { + entry = kv_response_state_read_from_pos(fp, id, hdr.tokens); + } + fclose(fp); + } + pthread_mutex_unlock(&s->kv_mu); + free(path); + free(key); + return entry; +} + +static response_state_entry *response_state_acquire_or_restore(server *s, + const char *id) { + response_state_entry *entry = response_state_acquire(s, id); + if (entry || !s || !id || !id[0]) return entry; + + response_state_entry *restored = response_state_read_checkpoint(s, id); + if (!restored) return NULL; + if (!response_state_insert(s, restored)) response_state_entry_free(restored); + return response_state_acquire(s, id); } static int live_text_prefix_prompt(server *s, server_slot *slot, @@ -9741,6 +11015,69 @@ static int responses_live_continuation_prompt(server *s, server_slot *slot, return live_tokens->len; } +/* previous_response_id continuation. + * + * A successful parser hit carries only the new Responses input as + * responses_state_tail_text. Prefer the producer slot when its epoch still + * names exactly the saved frontier; otherwise restore the record's private + * disk checkpoint when the optional KV cache retained it. */ +static int responses_state_continuation_prompt(server *s, server_slot *slot, + const request *req, int live_pos, + ds4_tokens *effective_prompt, + int *disk_cached_out, + char **disk_path_out, + uint8_t *disk_ext_out) { + if (!s || !slot || !req || !effective_prompt || !req->responses_stateful || + !req->responses_state || !req->responses_state_tail_text) return 0; + response_state_entry *entry = req->responses_state; + if (response_state_is_current(s, entry, slot, live_pos)) { + const ds4_tokens *live_tokens = ds4_session_tokens(slot->session); + if (live_tokens && live_tokens->len == entry->live_tokens) { + build_prompt_from_exact_prefix_and_token_suffix( + live_tokens, &req->prompt, effective_prompt); + return live_tokens->len; + } + } + + if (!response_state_checkpoint_saved(s, entry) || + !entry->checkpoint_key || !entry->checkpoint_key[0] || !s->kv.enabled) return 0; + + char *loaded_path = NULL; + uint8_t loaded_ext = 0; + int loaded = kv_cache_try_load_text(s, slot, entry->checkpoint_key, NULL, + &loaded_path, &loaded_ext, true, + KV_EXT_RESPONSES_ID); + bool exact = loaded == entry->live_tokens && + (loaded_ext & KV_EXT_RESPONSES_ID) != 0; + if (!exact) { + free(loaded_path); + if (loaded > 0) { + pthread_mutex_lock(&s->inference_mu); + ds4_session_invalidate(slot->session); + pthread_mutex_unlock(&s->inference_mu); + } + response_state_reset_checkpoint_cadence(slot); + return 0; + } + const ds4_tokens *live_tokens = ds4_session_tokens(slot->session); + if (!live_tokens || live_tokens->len != entry->live_tokens) { + free(loaded_path); + response_state_reset_checkpoint_cadence(slot); + return 0; + } + build_prompt_from_exact_prefix_and_token_suffix( + live_tokens, &req->prompt, effective_prompt); + /* A restarted slot has no in-memory response cadence. Seed it from the + * checkpoint's crossed boundary so the next short response does not stage + * the same full payload again. */ + response_state_seed_checkpoint_cadence(s, slot, entry->live_tokens); + if (disk_cached_out) *disk_cached_out = loaded; + if (disk_path_out) *disk_path_out = loaded_path; + else free(loaded_path); + if (disk_ext_out) *disk_ext_out = loaded_ext; + return live_tokens->len; +} + /* Tool-result Anthropic continuation. * * /v1/messages has no server-side response object like the OpenAI Responses @@ -10862,11 +12199,12 @@ static void canonicalize_tool_checkpoint(server *s, server_slot *slot, ds4_tokens effective = {0}; int loaded = kv_cache_try_load_text(s, slot, rendered.ptr ? rendered.ptr : "", - &effective, &path, NULL, false); + &effective, &path, NULL, false, 0); if (loaded == 0) { pthread_mutex_lock(&s->inference_mu); ds4_session_invalidate(slot->session); pthread_mutex_unlock(&s->inference_mu); + response_state_reset_checkpoint_cadence(slot); } char sync_err[160] = {0}; @@ -11201,38 +12539,60 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { const char *responses_live_match = NULL; int responses_live_match_ids = 0; int anthropic_live_match_ids = 0; - /* Responses gets the first chance to continue from live state. This is - * the whole point of the API shape: a request that is bound to prior live - * output by visible transcript or tool call ids does not need to prove an - * exact token-prefix match. Exact token/text/disk matching remains the - * fallback when the live state is absent or no longer describes the - * request. */ - int cached = responses_live_visible_prefix_prompt(s, slot, &j->req, old_pos, - &effective_prompt); - const char *cache_source = cached > 0 ? "responses-visible" : "none"; + int disk_cached = 0; + char *disk_cache_path = NULL; + uint8_t disk_cache_ext_flags = 0; + bool responses_state_continuation = false; + /* A response id is a stronger binding than visible replay or a tool id: + * its record names the exact KV frontier and the parser supplied only the + * new tail. Try it before any generic prefix matcher. */ + int cached = responses_state_continuation_prompt( + s, slot, &j->req, old_pos, &effective_prompt, + &disk_cached, &disk_cache_path, &disk_cache_ext_flags); + const char *cache_source = cached > 0 ? + (disk_cached > 0 ? "responses-id-disk" : "responses-id") : "none"; if (cached > 0) { - responses_live_match = "visible-prefix"; - if (responses_live_matches_request(s, slot, - &j->req.responses_live_call_ids, - old_pos)) - { - responses_live_match_ids = j->req.responses_live_call_ids.len; + responses_state_continuation = true; + prompt_for_sync = &effective_prompt; + } + if (cached == 0 && j->req.responses_stateful) { + ds4_tokens_free(&effective_prompt); + free(disk_cache_path); + http_error(j->fd, s->enable_cors, 409, + "previous_response_id is no longer available; replay the full input history"); + return; + } + /* Responses gets the next chance to continue from existing visible/tool + * live state. Exact token/text/disk matching remains the fallback for + * stateless replay and edited branches. */ + if (cached == 0) { + cached = responses_live_visible_prefix_prompt(s, slot, &j->req, old_pos, + &effective_prompt); + cache_source = cached > 0 ? "responses-visible" : "none"; + if (cached > 0) { + responses_live_match = "visible-prefix"; + if (responses_live_matches_request(s, slot, + &j->req.responses_live_call_ids, + old_pos)) + { + responses_live_match_ids = j->req.responses_live_call_ids.len; + } } } if (cached == 0) { cached = responses_live_continuation_prompt(s, slot, &j->req, old_pos, - &effective_prompt, - &responses_live_match_ids); + &effective_prompt, + &responses_live_match_ids); cache_source = cached > 0 ? "responses-tool-output" : "none"; if (cached > 0) responses_live_match = "tool-output-ids"; } - if (cached > 0) { + if (cached > 0 && !responses_state_continuation) { responses_live_continuation = true; prompt_for_sync = &effective_prompt; - } else { + } else if (cached == 0) { cached = anthropic_live_continuation_prompt(s, slot, &j->req, old_pos, - &effective_prompt, - &anthropic_live_match_ids); + &effective_prompt, + &anthropic_live_match_ids); if (cached > 0) { anthropic_live_continuation = true; cache_source = "anthropic-tool-output"; @@ -11287,9 +12647,6 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { prompt_for_sync = &effective_prompt; } } - int disk_cached = 0; - char *disk_cache_path = NULL; - uint8_t disk_cache_ext_flags = 0; if (cached == 0) { int text_cached = live_text_prefix_prompt(s, slot, &j->req, &effective_prompt); @@ -11299,6 +12656,14 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { prompt_for_sync = &effective_prompt; } } + /* A partial in-memory hit retains only an older shared prefix and will + * replace the live suffix during synchronization. Its response-id + * checkpoint cadence belongs to the old branch, not the new one. The + * response-id path is exempt: its disk restore seeded the cadence from the + * exact state record above. */ + if (cached > 0 && cached < old_pos && !responses_state_continuation) { + response_state_reset_checkpoint_cadence(slot); + } if (cached == 0 && old_pos > 0) { server_log(DS4_LOG_WARNING, "ds4-server: live kv cache miss%s live=%d prompt=%d common=%d reason=%s", @@ -11306,7 +12671,10 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { old_pos, j->req.prompt.len, common, trace_cache_miss_reason(&cache_diag)); } - if (cached == 0) slot->continued_last_store_tokens = 0; + if (cached == 0) { + slot->continued_last_store_tokens = 0; + response_state_reset_checkpoint_cadence(slot); + } if (s->kv.enabled && cached == 0 && old_pos >= s->kv.opt.min_tokens) { /* Loading a disk snapshot replaces the live Metal session. Persist the * current checkpoint first, otherwise a cache hit for an older prefix @@ -11325,8 +12693,7 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { } const bool responses_reasoning_state_preserved = cached > 0 && - ((!strcmp(cache_source, "responses-visible") || - !strcmp(cache_source, "responses-tool-output")) || + (responses_state_continuation || responses_live_continuation || (!strcmp(cache_source, "disk-text") && (disk_cache_ext_flags & KV_EXT_RESPONSES_VISIBLE))); const bool responses_visible_replay_without_reasoning = @@ -11334,6 +12701,13 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { j->req.responses_requires_live_reasoning && !responses_reasoning_state_preserved; const int prompt_tokens = prompt_for_sync->len; + if (prompt_tokens >= s->ctx_size) { + ds4_tokens_free(&effective_prompt); + free(disk_cache_path); + http_error_context_length_exceeded(j->fd, s->enable_cors, &j->req, + prompt_tokens, s->ctx_size); + return; + } /* OpenAI usage details: the reusable prefix is a cache read, while the * effective prompt suffix evaluated by ds4_session_sync() is written into * the live KV cache and can be reused by the next request. */ @@ -11496,6 +12870,7 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { ds4_tokens_free(&effective_prompt); return; } + response_state_advance_slot_epoch(s, slot); /* Once a non-live request wins, old protocol live bindings are stale. Keep * a binding only when this request explicitly continued from it. */ if (!responses_live_continuation) responses_live_clear(s, slot); @@ -11526,6 +12901,10 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { snprintf(id, sizeof(id), "%s-%llu", j->req.kind == REQ_CHAT ? "chatcmpl" : "cmpl", (unsigned long long)response_seq); + char responses_response_id[40] = {0}; + if (j->req.api == API_RESPONSES) { + responses_new_response_id(s, responses_response_id, sizeof(responses_response_id)); + } bool structured_stream = request_uses_structured_stream(&j->req); anthropic_stream anthropic_live = {0}; @@ -11581,7 +12960,7 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { } if (openai_live_chat) openai_stream_start(&j->req, &openai_live); if (responses_live_chat) { - responses_stream_init(&j->req, &responses_live); + responses_stream_init(&j->req, &responses_live, responses_response_id); responses_live.active = true; if (!responses_sse_created(j->fd, &j->req, &responses_live, responses_created_at)) { job_mark_cancelled(j); @@ -11871,6 +13250,7 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { pthread_mutex_lock(&s->inference_mu); ds4_session_invalidate(slot->session); pthread_mutex_unlock(&s->inference_mu); + response_state_reset_checkpoint_cadence(slot); stop_decode = true; break; } @@ -12160,7 +13540,8 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { parsed_reasoning, &parsed_calls, now_sec() - t0); if (j->req.api == API_RESPONSES) { - if (strcmp(final_finish, "error") && strcmp(final_finish, "length")) { + if (strcmp(final_finish, "error") && strcmp(final_finish, "length") && + !j->req.responses_stateful) { /* Store the post-turn visible transcript plus the live token * frontier. The next Responses request may replay only this * visible surface, while the real session also contains hidden @@ -12216,6 +13597,36 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { } bool response_ok = !job_cancelled(j); + response_state_entry *new_state = NULL; + /* Publish the state before the response id reaches the client, then keep + * the producer reference until the final write confirms the response was + * delivered. A failed write removes both the live entry and its checkpoint + * below, so a disconnected client cannot leave a continuation behind. */ + if (response_ok && j->req.api == API_RESPONSES && responses_response_id[0] && + strcmp(final_finish, "error")) + { + const ds4_tokens *response_tokens = ds4_session_tokens(slot->session); + new_state = response_tokens ? + response_state_remember(s, slot->id, response_tokens->len, + response_state_slot_epoch(s, slot), responses_response_id, + &j->req, &parsed_calls) : NULL; + if (new_state) { + bool checkpoint_saved = false; + const int checkpoint_boundary = response_tokens ? + response_state_checkpoint_boundary(s, slot, response_tokens->len) : 0; + if (checkpoint_boundary > 0) { + /* Note the boundary before staging. A disk-full or oversize + * outcome is still an attempted cadence point, so the next + * short response remains live-only instead of retrying a full + * checkpoint synchronously. */ + response_state_note_checkpoint_attempt(slot, checkpoint_boundary); + checkpoint_saved = kv_cache_store_response_state_prefix( + s, slot, response_tokens, new_state); + } + response_state_set_checkpoint_saved(s, new_state, checkpoint_saved); + } + } + if (job_cancelled(j)) response_ok = false; if (response_ok && j->req.stream) { if (j->req.api == API_ANTHROPIC) { response_ok = anthropic_sse_finish_live(j->fd, s, &j->req, id, &anthropic_live, @@ -12256,7 +13667,8 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { &parsed_calls, final_finish, prompt_tokens, completion); } else if (response_ok && j->req.api == API_RESPONSES) { - response_ok = responses_final_response(j->fd, s->enable_cors, &j->req, id, + response_ok = responses_final_response(j->fd, s->enable_cors, &j->req, + responses_response_id, parsed_content ? parsed_content : (text.ptr ? text.ptr : ""), parsed_reasoning, &parsed_calls, final_finish, @@ -12270,6 +13682,11 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { } if (job_cancelled(j)) response_ok = false; if (!response_ok) { + if (new_state) { + kv_cache_discard_response_state_checkpoint(s, new_state); + response_state_forget(s, new_state); + response_state_reset_checkpoint_cadence(slot); + } job_mark_cancelled(j); final_finish = "error"; snprintf(err, sizeof(err), "client disconnected"); @@ -12281,6 +13698,7 @@ static void generate_job_inner(server *s, server_slot *slot, job *j) { req_flags[0] ? " " : "", req_flags); } + if (new_state) response_state_release(s, new_state); if (j->req.kind == REQ_CHAT && j->req.has_tools) { char flags[80]; log_flags(flags, sizeof(flags), @@ -12383,6 +13801,13 @@ static bool live_state_contains_all(const live_tool_state *state, static int job_required_slot_locked(server *s, const job *j) { if (!s || !j) return -1; const request *r = &j->req; + if (r->responses_stateful && r->responses_state && + r->responses_state->indexed && + r->responses_state->slot_id >= 0 && + r->responses_state->slot_id < s->slot_count) + { + return r->responses_state->slot_id; + } for (int i = 0; i < s->slot_count; i++) { server_slot *slot = &s->slots[i]; if (r->responses_requires_live_tool_state && @@ -12855,7 +14280,7 @@ static void *client_main(void *arg) { free(req.model); req.model = xstrdup(server_model_id_from_engine(s->engine)); } - if (request_exceeds_context(&req, ctx_size)) { + if (!req.responses_stateful && request_exceeds_context(&req, ctx_size)) { http_error_context_length_exceeded(fd, s->enable_cors, &req, req.prompt.len, ctx_size); request_free(&req); goto done; @@ -12946,6 +14371,7 @@ typedef struct { bool kv_cache_reject_different_quant; bool disable_exact_dsml_tool_replay; int tool_memory_max_ids; + int response_state_max_ids; bool enable_cors; int batched_sessions; int mixed_prefill_quantum; @@ -13020,6 +14446,7 @@ static void server_close_resources(server *s) { s->trace = NULL; } kv_cache_close(&s->kv); + response_state_index_free(&s->response_states); tool_memory_free(&s->tool_mem); for (int i = 0; i < s->slot_count; i++) { server_slot *slot = &s->slots[i]; @@ -13087,6 +14514,7 @@ static server_config parse_options(int argc, char **argv) { .ctx_size = 32768, .default_tokens = 393216, .tool_memory_max_ids = DS4_TOOL_MEMORY_DEFAULT_MAX_IDS, + .response_state_max_ids = DS4_RESPONSE_STATE_DEFAULT_MAX_IDS, .mixed_prefill_quantum = 128, }; c.kv_cache = kv_cache_default_options(); @@ -13181,6 +14609,8 @@ static server_config parse_options(int argc, char **argv) { c.disable_exact_dsml_tool_replay = true; } else if (!strcmp(arg, "--tool-memory-max-ids")) { c.tool_memory_max_ids = parse_int_arg(need_arg(&i, argc, argv, arg), arg); + } else if (!strcmp(arg, "--response-state-max-ids")) { + c.response_state_max_ids = parse_int_arg(need_arg(&i, argc, argv, arg), arg); } else if (!strcmp(arg, "--quality")) { c.engine.quality = true; } else if (!strcmp(arg, "--ssd-streaming")) { @@ -13382,6 +14812,7 @@ int main(int argc, char **argv) { s.default_tokens = cfg.default_tokens; s.disable_exact_dsml_tool_replay = cfg.disable_exact_dsml_tool_replay; s.tool_mem.max_entries = cfg.tool_memory_max_ids; + s.response_states.max_entries = cfg.response_state_max_ids; s.enable_cors = cfg.enable_cors; s.slots = xmalloc((size_t)slot_count * sizeof(*s.slots)); memset(s.slots, 0, (size_t)slot_count * sizeof(*s.slots)); @@ -13610,6 +15041,13 @@ static void test_mixed_prefill_quantum_option(void) { char *default_argv[] = {"ds4-server"}; server_config defaults = parse_options(1, default_argv); TEST_ASSERT(defaults.mixed_prefill_quantum == 128); + TEST_ASSERT(defaults.response_state_max_ids == DS4_RESPONSE_STATE_DEFAULT_MAX_IDS); + + char *response_state_argv[] = { + "ds4-server", "--response-state-max-ids", "32" + }; + server_config response_state_cfg = parse_options(3, response_state_argv); + TEST_ASSERT(response_state_cfg.response_state_max_ids == 32); char *custom_argv[] = { "ds4-server", "--mixed-prefill-quantum", "2048" @@ -14420,18 +15858,29 @@ static void test_responses_usage_reports_cache_details(void) { r.cache_read_tokens = 7; r.cache_write_tokens = 3; + server id_server = {0}; + pthread_mutex_init(&id_server.tool_mu, NULL); + char response_id[40] = {0}; + responses_new_response_id(&id_server, response_id, sizeof(response_id)); + TEST_ASSERT(!strncmp(response_id, "resp_", 5)); + int sv[2]; TEST_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0); if (sv[0] < 0 || sv[1] < 0) { + pthread_mutex_destroy(&id_server.tool_mu); request_free(&r); return; } - TEST_ASSERT(responses_final_response(sv[0], false, &r, "resp_usage", "OK", NULL, NULL, + TEST_ASSERT(responses_final_response(sv[0], false, &r, response_id, "OK", NULL, NULL, "stop", 10, 2)); shutdown(sv[0], SHUT_WR); char *out = read_socket_text(sv[1]); + char expected_response_id[64]; + snprintf(expected_response_id, sizeof(expected_response_id), + "\"id\":\"%s\"", response_id); + TEST_ASSERT(strstr(out, expected_response_id) != NULL); TEST_ASSERT(strstr(out, "\"usage\":{\"input_tokens\":10") != NULL); TEST_ASSERT(strstr(out, "\"input_tokens_details\":{") != NULL); TEST_ASSERT(strstr(out, "\"cached_tokens\":7") != NULL); @@ -14445,18 +15894,20 @@ static void test_responses_usage_reports_cache_details(void) { TEST_ASSERT(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0); if (sv[0] < 0 || sv[1] < 0) { + pthread_mutex_destroy(&id_server.tool_mu); request_free(&r); return; } responses_stream st; - responses_stream_init(&r, &st); + responses_stream_init(&r, &st, response_id); TEST_ASSERT(responses_sse_completed(sv[0], &r, &st, NULL, NULL, "stop", 10, 2, 1234)); shutdown(sv[0], SHUT_WR); out = read_socket_text(sv[1]); TEST_ASSERT(strstr(out, "\"type\":\"response.completed\"") != NULL); + TEST_ASSERT(strstr(out, expected_response_id) != NULL); TEST_ASSERT(strstr(out, "\"usage\":{\"input_tokens\":10") != NULL); TEST_ASSERT(strstr(out, "\"input_tokens_details\":{") != NULL); TEST_ASSERT(strstr(out, "\"cached_tokens\":7") != NULL); @@ -14468,6 +15919,7 @@ static void test_responses_usage_reports_cache_details(void) { responses_stream_free(&st); close(sv[0]); close(sv[1]); + pthread_mutex_destroy(&id_server.tool_mu); request_free(&r); } @@ -16291,6 +17743,263 @@ static void test_responses_tool_output_id_validation(void) { pthread_mutex_destroy(&s.tool_mu); } +static void test_responses_response_state_index(void) { + server s = {0}; + pthread_mutex_init(&s.tool_mu, NULL); + s.response_states.max_entries = 2; + + request base; + request_init(&base, REQ_CHAT, 128); + base.api = API_RESPONSES; + base.model_syntax = SERVER_MODEL_SYNTAX_DEEPSEEK; + base.think_mode = DS4_THINK_HIGH; + base.has_tools = true; + base.cache_read_tokens = 77; + responses_request_set_state_config(&base, "system", "tools-v1", "tools-v1", false); + base.responses_state_fingerprint = response_state_fingerprint_values( + base.model_syntax, base.think_mode, base.has_tools, + base.responses_state_tool_choice_none, base.model, + base.responses_state_instructions, base.responses_state_request_tools, + base.responses_state_active_tools); + + tool_calls calls = {0}; + tool_call call = { + .id = xstrdup("call_saved"), + .name = xstrdup("exec"), + .arguments = xstrdup("{}"), + }; + tool_calls_push(&calls, call); + + response_state_entry *saved = response_state_remember( + &s, 2, 77, 9, "resp_saved", &base, &calls); + TEST_ASSERT(saved != NULL); + TEST_ASSERT(saved && saved->fingerprint == base.responses_state_fingerprint); + TEST_ASSERT(saved && saved->live_tokens == 77); + TEST_ASSERT(saved && saved->prompt_prefix_tokens == 77); + TEST_ASSERT(saved && saved->frontier_epoch == 9); + TEST_ASSERT(saved && !strcmp(saved->checkpoint_key, "responses-id:resp_saved")); + + request follow; + request_init(&follow, REQ_CHAT, 128); + follow.api = API_RESPONSES; + follow.model_syntax = SERVER_MODEL_SYNTAX_DEEPSEEK; + follow.think_mode = DS4_THINK_HIGH; + TEST_ASSERT(response_state_config_matches(saved, &follow)); + follow.responses_state_instructions_set = true; + responses_request_set_state_config(&follow, "edited", "", "", false); + TEST_ASSERT(!response_state_config_matches(saved, &follow)); + follow.responses_state_instructions_set = false; + free(follow.model); + follow.model = xstrdup("other-model"); + follow.model_from_request = true; + TEST_ASSERT(!response_state_config_matches(saved, &follow)); + request_free(&follow); + + chat_msgs delta = {0}; + chat_msg tool = {0}; + tool.role = xstrdup("tool"); + tool.content = xstrdup("ok"); + chat_msg_add_tool_call_id(&tool, "call_saved"); + chat_msgs_push(&delta, tool); + char err[160] = {0}; + TEST_ASSERT(responses_input_is_append_only(&delta)); + TEST_ASSERT(responses_validate_state_tool_outputs( + &delta, response_state_call_ids(saved), err, sizeof(err))); + chat_msgs unbound = {0}; + chat_msg unbound_tool = {0}; + unbound_tool.role = xstrdup("tool"); + unbound_tool.content = xstrdup("unbound"); + chat_msgs_push(&unbound, unbound_tool); + err[0] = '\0'; + TEST_ASSERT(!responses_validate_state_tool_outputs( + &unbound, response_state_call_ids(saved), err, sizeof(err))); + TEST_ASSERT(strstr(err, "requires a call_id") != NULL); + chat_msgs_free(&unbound); + free(delta.v[0].tool_call_id); + delta.v[0].tool_call_id = xstrdup("call_edited"); + err[0] = '\0'; + TEST_ASSERT(!responses_validate_state_tool_outputs( + &delta, response_state_call_ids(saved), err, sizeof(err))); + TEST_ASSERT(strstr(err, "edited branch") != NULL); + chat_msg assistant = {0}; + assistant.role = xstrdup("assistant"); + assistant.content = xstrdup("replayed"); + chat_msgs_push(&delta, assistant); + TEST_ASSERT(!responses_input_is_append_only(&delta)); + chat_msgs_free(&delta); + + server_slot slots[3] = {0}; + slots[2].id = 2; + slots[2].frontier_epoch = 9; + s.slots = slots; + s.slot_count = 3; + TEST_ASSERT(response_state_is_current(&s, saved, &slots[2], 77)); + slots[2].frontier_epoch++; + TEST_ASSERT(!response_state_is_current(&s, saved, &slots[2], 77)); + slots[2].frontier_epoch--; + ds4_tokens prefix = {0}, tail = {0}, joined = {0}; + ds4_tokens_push(&prefix, 7); + ds4_tokens_push(&prefix, 11); + ds4_tokens_push(&tail, 13); + ds4_tokens_push(&tail, 17); + build_prompt_from_exact_prefix_and_token_suffix(&prefix, &tail, &joined); + TEST_ASSERT(joined.len == 4); + TEST_ASSERT(joined.v[0] == 7 && joined.v[1] == 11 && + joined.v[2] == 13 && joined.v[3] == 17); + ds4_tokens_free(&prefix); + ds4_tokens_free(&tail); + ds4_tokens_free(&joined); + job j = {0}; + j.req.responses_stateful = true; + j.req.responses_state = saved; + pthread_mutex_lock(&s.tool_mu); + TEST_ASSERT(job_required_slot_locked(&s, &j) == 2); + pthread_mutex_unlock(&s.tool_mu); + j.req.responses_state = NULL; + + /* A response whose final write fails must not remain resumable while its + * producer still owns the pinned record. */ + response_state_entry *abandoned = response_state_remember( + &s, 1, 88, 10, "resp_abandoned", &base, NULL); + TEST_ASSERT(abandoned != NULL); + response_state_forget(&s, abandoned); + TEST_ASSERT(response_state_acquire(&s, "resp_abandoned") == NULL); + response_state_release(&s, abandoned); + + response_state_release(&s, saved); + response_state_entry *second = response_state_remember( + &s, 1, 88, 10, "resp_second", &base, NULL); + TEST_ASSERT(second != NULL); + response_state_release(&s, second); + response_state_entry *third = response_state_remember( + &s, 0, 99, 11, "resp_third", &base, NULL); + TEST_ASSERT(third != NULL); + response_state_release(&s, third); + TEST_ASSERT(response_state_acquire(&s, "resp_saved") == NULL); + response_state_entry *kept = response_state_acquire(&s, "resp_second"); + TEST_ASSERT(kept != NULL); + response_state_release(&s, kept); + + tool_calls_free(&calls); + request_free(&base); + response_state_index_free(&s.response_states); + pthread_mutex_destroy(&s.tool_mu); +} + +static void test_responses_response_state_checkpoint_trailer(void) { + server writer = {0}; + pthread_mutex_init(&writer.tool_mu, NULL); + + request base; + request_init(&base, REQ_CHAT, 128); + base.api = API_RESPONSES; + base.model_syntax = SERVER_MODEL_SYNTAX_DEEPSEEK; + base.think_mode = DS4_THINK_HIGH; + base.has_tools = true; + base.cache_read_tokens = 123; + responses_request_set_state_config(&base, "system", "tools-v1", "tools-v1", false); + tool_schema_order order = { + .name = xstrdup("exec"), + .wire_name = xstrdup("shell"), + .namespace = xstrdup("local_"), + }; + tool_schema_order_prop_push(&order, xstrdup("command")); + tool_schema_orders_push(&base.tool_orders, order); + tool_calls calls = {0}; + tool_call call = { + .id = xstrdup("call_checkpoint"), + .name = xstrdup("exec"), + .arguments = xstrdup("{\"command\":\"pwd\"}"), + }; + tool_calls_push(&calls, call); + response_state_entry *saved = response_state_make( + "resp_checkpoint", -1, 123, 0, &base, &calls); + TEST_ASSERT(saved != NULL); + + char dir[] = "/tmp/ds4-response-state-XXXXXX"; + TEST_ASSERT(mkdtemp(dir) != NULL); + char sha[41]; + ds4_kvstore_sha1_bytes_hex(saved->checkpoint_key, + strlen(saved->checkpoint_key), sha); + ds4_kvstore disk = {.enabled = true, .dir = dir}; + char *path = ds4_kvstore_path_for_sha(&disk, sha); + FILE *fp = fopen(path, "wb"); + TEST_ASSERT(fp != NULL); + uint8_t header[KV_CACHE_FIXED_HEADER]; + ds4_kvstore_fill_header(header, 0, 2, KV_REASON_CONTINUED, + KV_EXT_RESPONSES_ID | KV_EXT_TOOL_MAP, + (uint32_t)saved->live_tokens, 0, 256, 1, 1, 0); + uint8_t text_len[4]; + le_put32(text_len, (uint32_t)strlen(saved->checkpoint_key)); + TEST_ASSERT(fwrite(header, 1, sizeof(header), fp) == sizeof(header)); + TEST_ASSERT(fwrite(text_len, 1, sizeof(text_len), fp) == sizeof(text_len)); + TEST_ASSERT(fwrite(saved->checkpoint_key, 1, strlen(saved->checkpoint_key), fp) == + strlen(saved->checkpoint_key)); + kv_response_state_trailer trailer = {.server = &writer, .entry = saved}; + uint64_t trailer_bytes = 0; + TEST_ASSERT(kv_response_state_trailer_write_cb( + &trailer, fp, saved->checkpoint_key, &trailer_bytes)); + TEST_ASSERT(trailer_bytes > KV_RESPONSE_STATE_HEADER + KV_TOOL_MAP_HEADER); + TEST_ASSERT(fclose(fp) == 0); + + server restarted = {0}; + restarted.ctx_size = 256; + restarted.kv.enabled = true; + restarted.kv.dir = xstrdup(dir); + pthread_mutex_init(&restarted.tool_mu, NULL); + pthread_mutex_init(&restarted.kv_mu, NULL); + response_state_entry *restored = response_state_acquire_or_restore( + &restarted, "resp_checkpoint"); + TEST_ASSERT(restored != NULL); + TEST_ASSERT(restored && restored->checkpoint_saved); + TEST_ASSERT(restored && restored->slot_id == -1 && restored->live_tokens == 123); + TEST_ASSERT(restored && response_state_config_matches(restored, &base)); + TEST_ASSERT(restored && id_list_contains(response_state_call_ids(restored), + "call_checkpoint")); + TEST_ASSERT(restored && restored->tool_orders.len == 1); + const tool_schema_order *restored_order = restored ? + tool_schema_orders_find(&restored->tool_orders, "exec") : NULL; + TEST_ASSERT(restored_order && !strcmp(restored_order->wire_name, "shell")); + TEST_ASSERT(restored_order && !strcmp(restored_order->namespace, "local_")); + TEST_ASSERT(restored_order && restored_order->len == 1 && + !strcmp(restored_order->prop[0], "command")); + response_state_release(&restarted, restored); + + response_state_index_free(&restarted.response_states); + pthread_mutex_destroy(&restarted.kv_mu); + pthread_mutex_destroy(&restarted.tool_mu); + free(restarted.kv.dir); + TEST_ASSERT(unlink(path) == 0); + TEST_ASSERT(rmdir(dir) == 0); + free(path); + response_state_entry_free(saved); + tool_calls_free(&calls); + request_free(&base); + pthread_mutex_destroy(&writer.tool_mu); +} + +static void test_response_state_checkpoint_cadence(void) { + server s = {0}; + server_slot slot = {0}; + s.kv.enabled = true; + s.kv.opt = kv_cache_default_options(); + s.kv.opt.min_tokens = 512; + s.kv.opt.continued_interval_tokens = 8192; + s.kv.opt.boundary_align_tokens = 2048; + + const int first = response_state_checkpoint_boundary(&s, &slot, 8198); + TEST_ASSERT(first == 8192); + response_state_note_checkpoint_attempt(&slot, first); + TEST_ASSERT(slot.response_state_last_checkpoint_tokens == 8192); + TEST_ASSERT(response_state_checkpoint_boundary(&s, &slot, 8200) == 0); + TEST_ASSERT(response_state_checkpoint_boundary(&s, &slot, 16383) == 0); + TEST_ASSERT(response_state_checkpoint_boundary(&s, &slot, 16384) == 16384); + + response_state_reset_checkpoint_cadence(&slot); + response_state_seed_checkpoint_cadence(&s, &slot, 8198); + TEST_ASSERT(slot.response_state_last_checkpoint_tokens == 8192); +} + static void test_responses_stateless_tool_replay_requires_reasoning(void) { server s = {0}; server_slot slot; @@ -18378,6 +20087,9 @@ static void ds4_server_unit_tests_run(void) { test_tool_checkpoint_canonicalization_gate_exact_replay(); test_responses_live_tail_renders_tool_outputs_only(); test_responses_tool_output_id_validation(); + test_responses_response_state_index(); + test_responses_response_state_checkpoint_trailer(); + test_response_state_checkpoint_cadence(); test_responses_stateless_tool_replay_requires_reasoning(); test_responses_visible_suffix_matches_client_replay(); test_exact_dsml_tool_replay_can_be_disabled(); diff --git a/misc/RESPONSE_API.md b/misc/RESPONSE_API.md index b879f89ba5..da81670df6 100644 --- a/misc/RESPONSE_API.md +++ b/misc/RESPONSE_API.md @@ -1,447 +1,147 @@ -# Responses API Continuation Plan +# Responses API continuation state -This note tracks the design for fixing `/v1/responses` tool-call continuation -and the KV-cache behavior around it. +`/v1/responses` supports two ways to continue a turn: -## Problem +- **Stateless replay**: send the complete visible `input` history. DS4 renders + and tokenizes that history, then uses the normal token/text/KV prefix paths. +- **Response-id continuation**: send the preceding `resp_` value as + `previous_response_id` and send only the new input items. DS4 resolves the + saved exact model frontier and renders/tokenizes only the continuation tail. -The current implementation handles `/v1/responses` too much like a stateless -chat-completions replay API. After a tool call, it tries to canonicalize the -live KV checkpoint so it matches the next rendered prompt byte-for-byte. +The second form is intended for clients that would otherwise resend every +visible Responses item on every turn. It keeps generated hidden reasoning and +sampled tool-call text in the saved checkpoint rather than attempting to +recreate them from the visible replay. -That is wrong for the live Responses protocol path. +## Supported request shape -In a live Responses tool loop, the next request is not an unrelated prompt. It -is a continuation bound to the previous model output by tool-call ids. The live -KV already contains the true assistant turn, including hidden reasoning that -the client may not replay. Rebuilding the session to match only the visible -client transcript is both slow and less faithful to the state the model actually -produced. +A completed response returns an opaque `resp_` id. A continuation can look like: -Observed symptom: - -```text -tool checkpoint canonicalization needs rebuild ... common=16846 live=16994 canonical=16937 -tool checkpoint canonicalized ... via=rebuild +```json +{ + "previous_response_id": "resp_example", + "input": [ + {"role": "user", "content": "Now summarize that in one sentence."} + ] +} ``` -This caused a long pause because DS4 rebuilt roughly the whole context before -returning the streamed tool-call response tail. - -## Protocol Model - -There are two useful id classes: - -- `previous_response_id` / `conversation`: response-level server-side state. DS4 - does not currently persist this. If a non-null value arrives, return an error - asking the client to replay full input. -- `call_id`: tool-call binding. A `function_call_output` or hosted-tool output - with this id is the continuation of the previous assistant tool call. - -The server should distinguish two modes: - -- Live continuation: the request contains tool outputs for tool-call ids still - known by the live server state. Continue from the live KV state and append the - new tool-result suffix. -- Stateless replay: the request includes the full history. Render and match the - best prefix, using exact DSML replay from tool-memory when possible, just like - chat-completions style replay. - -Unknown tool-output ids are only valid if the request also includes the matching -prior function-call item in the replayed history. If neither live state nor full -history can explain the id, return an error. - -Core mental model: - -- The Responses API is designed so a server can avoid matching an already-known - prefix. If the request is tied to live server state by response/conversation - state or by immediately returning tool outputs for live call ids, DS4 should - continue from the live KV instead of proving that the client-visible replay - tokenizes to the same prefix. -- Prefix matching is only the fallback for stateless replay, cold start, - server restart, branch/edit, or multi-client cases where the server no longer - has the relevant live state. -- The fallback should reuse the techniques learned for chat continuation: - exact token-prefix match first, then rendered string-prefix match with - suffix retokenization, then disk string-prefix checkpoints. The adaptation - for Responses is that reasoning/tool-call stateless replay must include the - reasoning state the protocol requires, not just the visible transcript. - -## Official OpenAI Docs Findings - -Sources checked: - -- Conversation state: - `https://developers.openai.com/api/docs/guides/conversation-state` -- Function calling: - `https://developers.openai.com/api/docs/guides/function-calling` -- Responses API reference: - `https://platform.openai.com/docs/api-reference/responses` - -Findings that affect the implementation: - -- The Responses API has two explicit server-side state mechanisms: - `previous_response_id` and `conversation`. - - `previous_response_id` chains a new response to a stored prior response. - - `conversation` prepends persisted conversation items and then appends new - input/output items after the response completes. - - They are mutually exclusive in the API reference. -- Stateless/manual Responses state is also valid, but the client must preserve - the model output items. The function-calling guide shows: - - Start with an `input` list. - - Call `responses.create()`. - - Append `response.output` to the same input list. - - Append `function_call_output` items with the matching `call_id`. - - Send the resulting input list back. -- For reasoning models, the function-calling guide explicitly says that any - reasoning items returned in model responses with tool calls must also be - passed back with tool-call outputs. -- The Responses API reference exposes `include: - ["reasoning.encrypted_content"]`, described as enabling reasoning items to be - used in stateless multi-turn conversations when `store=false` or ZDR applies. - -Implication for DS4: - -- A request containing only a function-call output is not enough to recreate - stateless reasoning context unless DS4 still has live state for that call id. -- A full stateless replay should include the prior function/custom tool call - item and, for reasoning/tool-call turns, the reasoning item or an equivalent - opaque reasoning-state item. -- DS4 currently does not implement durable `previous_response_id` or - `conversation` state, so non-null values should remain rejected unless we add - a real response/conversation store. -- The current Codex-style `store=false` / `include=[]` flow can still work as a - live local server fast path, but if the live state is gone DS4 should reject - and ask for a full replay that includes the missing reasoning state, rather - than silently rebuilding a prompt with hidden reasoning removed. - -## Current Code Facts - -- Exact DSML replay via `tool_memory` / rax is still valid and working. It fixes - the visible tool-call block by replaying the sampled DSML for known call ids. -- The bad path is `canonicalize_tool_checkpoint()` for Responses tool calls when - reasoning summaries are not emitted. It drops hidden reasoning and can trigger - full rebuilds. -- `live_text_prefix_prompt()` is still useful for ordinary replay/cache matching, - but it is not enough for the live Responses continuation case because the - client-visible text can omit hidden reasoning that exists in the live KV. -- `ds4_session_save_snapshot()` / `ds4_session_load_snapshot()` exist, but they - should not be the primary continuation mechanism for this protocol path. - -## Implementation Plan - -1. Track live Responses continuation state. - - Remember the latest Responses assistant tool-call turn after generation. - - Store the generated call ids and enough rendered visible text to recognize - the next request. - - Keep the live KV as authoritative; do not canonicalize away hidden - reasoning for this path. - - Treat this as an optimization for the current live single-server session, - not as a substitute for `previous_response_id` / `conversation`. - - Prefer this path for Codex/pi tool loops when integration tests show those - clients send tool outputs that are directly bound to the previous live - call ids. - -2. Parse Responses tool outputs with validation. - - Collect function/custom/hosted tool-output ids during `parse_responses_input()`. - - Collect function/custom/hosted call ids present in the replayed history. - - For reasoning-mode tool-call turns, distinguish a true full replay from a - visible-only replay. A matching prior call id without the reasoning item is - enough to render visible history, but not enough to reconstruct hidden - model state if live continuation is unavailable. - - Reject tool-output ids that are neither known live ids nor present as prior - calls in the same request history. - -3. Add a live continuation fast path before normal cache matching. - - If the request is a Responses request and its first new semantic item is a - tool output for the remembered live call ids, build only the suffix that - must be appended to the current live KV. - - Tokenize that suffix independently from the live token prefix, as the - existing text-prefix path does, to avoid BPE boundary mistakes. - - Set `prompt_for_sync` to `live_tokens + suffix_tokens`, with - `cached = live_tokens->len`. - -4. Keep stateless replay behavior. - - If the request is not a live continuation, render the full prompt and use - the existing token-prefix, live text-prefix, and disk text-prefix cache - paths. - - Exact DSML tool replay remains available to improve matching. - - If the request is a stateless reasoning/tool-call replay without reasoning - items or equivalent opaque reasoning state, reject instead of pretending the - visible transcript is complete. - -5. Remove invalid Responses canonicalization. - - Do not call `canonicalize_tool_checkpoint()` for Responses live tool-call - continuations. - - Remove the Responses-specific reasoning-dropping behavior in - `build_tool_checkpoint_suffix()`. - - Keep or narrow toolless thinking canonicalization only where the next - request is truly a stateless replay and the live hidden reasoning cannot be - continued by protocol ids. - -6. Improve errors. - - Non-null `previous_response_id` / `conversation`: return a clear 400 error - unless DS4 later implements persistent response state. - - Unknown tool-output id without a matching prior function call in the same - request: return a clear 400 error explaining that DS4 needs full history. - - Reasoning/tool-call replay that lacks reasoning state and is not live: - return a clear 400 explaining that DS4 needs either live continuation or a - full Responses replay with reasoning items / encrypted reasoning content. - -## Tests - -Automated tests: - -- Responses live tool continuation keeps hidden reasoning in live KV and appends - only the tool-result suffix. -- Responses live continuation does not call tool checkpoint canonicalization. -- Unknown tool-output id with no prior function call is rejected. -- Unknown tool-output id with a prior function call in replayed history is - accepted as stateless replay only when the replay has enough reasoning state - for the selected mode. -- Reasoning/tool-call stateless replay without reasoning state is rejected if - live continuation is unavailable. -- Normal chat-completions exact DSML replay still works. -- Disk KV tool-map restore still works for stateless replay. -- BPE boundary handling remains correct for live-prefix plus text suffix. - -Live tests: - -- Codex multi-turn session with repeated tool calls. - - Expected: after each tool call, next request reports a memory-token - continuation or equivalent live continuation, not a rebuild. - - Expected: no long pause after `finish=tool_calls`. - - Verify the actual client behavior: whether Codex sends `previous_response_id`, - full `response.output`, function-call outputs only, reasoning items, or some - mixture. -- Pi agent multi-turn session with tools. - - Expected: tool calls execute and the agent recovers normally from tool - outputs. - - Expected: no unexpected cache misses. - - Verify the actual client behavior independently from Codex. Do not assume - both clients use the same Responses replay style. -- Stateless replay request with full history and known call ids. - - Expected: exact DSML replay via rax and best-prefix cache matching. -- Bad continuation request with only a tool output for an unknown id. - - Expected: HTTP 400 with a direct "replay full history" style message. - -## Current Progress - -- Confirmed from trace that the pause is synchronous checkpoint - canonicalization and rebuild after a parsed tool call. -- Confirmed rax exact DSML replay is not the failing part: - `tool_replay: mem=4 disk=0 canonical=0 missing_ids=0`. -- Confirmed the current Responses parser rejects non-null - `previous_response_id` / `conversation`, which is the right behavior until - server-side response persistence exists. -- Checked official OpenAI documentation. The plan was updated to account for - the documented requirement that reasoning-model tool-call loops preserve - reasoning items in stateless manual replay. -- Implemented live Responses continuation state and removed Responses tool-call - checkpoint canonicalization from the main path. -- Added normal server logs for accepted live Responses continuations: - `responses live continuation match=... cached=... prompt=...`. - Tool-output continuations now also report the matched id count, for example - `match=visible-prefix ids=1`. -- Added Responses-visible disk checkpoint keys. The payload is still the exact - live KV with hidden reasoning, but the file name / lookup text is the visible - transcript the client can replay. This is required when switching between - Codex, pi, and synthetic clients, because only one live KV session can be - resident at a time. - -## QA Log - -All tests in this section were run on the local M3 Max server with Metal, -`--ctx 65536`, and `/v1/responses` clients configured for -`deepseek-v4-flash`. - -Automated checks: - -- `make ds4_test && ./ds4_test --server && make ds4-server`: pass. -- Added unit coverage for unknown tool-output ids, reasoning-required - stateless replay, Responses live-tail rendering, and final-answer visible - suffix handling. - -Official protocol checks: - -- `previous_response_id != null`: returns HTTP 400 with - `previous_response_id is not supported; replay full input instead`. -- `conversation != null`: returns HTTP 400 with - `conversation is not supported; replay full input instead`. -- Tool output for an unknown `call_id`: returns HTTP 400 with - `unknown tool output call_id ...; replay full Responses history`. -- Stateless replay with a prior tool call but no reasoning item: returns HTTP - 400 with `Responses replay is missing reasoning state...`. -- Stateless replay with a prior tool call and an explicit reasoning item: - returns HTTP 200 and generates from the replayed history. - -Codex live behavior: - -- Codex sends `store=false`, `include=[]`, `prompt_cache_key=`, and - full visible replay. -- Exact token matching fails at hidden thinking, as expected: - `live_prompt_common` stops where live KV has hidden reasoning and the visible - replay has ``. -- Live visible continuation now catches this path: - `cache_source: responses-visible`, with short suffix prefill after tool - outputs and after normal resumed user turns. -- Repeated Codex turns completed with tool execution and no - `canonicalization needs rebuild` pause. - -Pi live behavior: - -- Pi uses a custom local provider with `api: openai-responses`. -- Pi sends `reasoning.summary=auto` and `include: - ["reasoning.encrypted_content"]`. -- Pi replays reasoning summaries for assistant tool-call turns, but not for - final assistant text answers. The visible-prefix state must therefore include - tool-call reasoning summaries when present, but omit final-answer hidden - reasoning. -- Pi first turn, tool result, resumed user turn, and resumed tool result all - completed. Accepted live continuations print: - `responses live continuation match=visible-prefix ...`; tool-output turns - print `ids=1` in the normal server log. - -Interleaved session switching: - -- Sequence tested: Codex turn, pi turn, Codex resume, pi resume, Codex resume, - plus synthetic curl traffic between agent turns. -- On switching clients, the live KV miss is expected and logged in orange. -- Before replacing live state, DS4 stores the outgoing Responses session as - `key=responses-visible`. -- Returning to Codex after pi loaded the Codex checkpoint from disk: - `kv cache hit text tokens=11772 ...`, then prefills only the new suffix. -- Returning to pi after Codex loaded the pi checkpoint from disk: - `kv cache hit text tokens=1654 ...`, then prefills only the new suffix. -- Returning to Codex after synthetic curl traffic again loaded the latest Codex - checkpoint from disk: - `kv cache hit text tokens=12211 ...`. -- The earlier failure mode was reproduced before the fix: Codex resume after - pi received `Responses replay is missing reasoning state...` because the - guard ran before disk recovery and the disk key was raw hidden text. The fix - is to key Responses checkpoints by visible transcript and accept only - `KV_EXT_RESPONSES_VISIBLE` disk hits for missing-reasoning replay. - -Remaining risks / follow-up: - -- DS4 still does not implement durable `previous_response_id` or - `conversation`; rejecting non-null values remains intentional. -- `reasoning.encrypted_content` is not decrypted. It is useful for real OpenAI - stateless clients, but DS4 currently relies on live state or visible-key disk - checkpoints for hidden reasoning recovery. -- The tool-output-only continuation path is unit-tested, but the observed Codex - and pi clients both used full visible replay in their normal runs. -- Tool-output-only continuation was also exercised directly with curl: - first request produced `call_3c4891795bf43aca507f4a987707b259`, second - request sent only `function_call_output` for that id, and the server accepted - it as `responses live continuation match=tool-output-ids ids=1`. -- Code review follow-up tightened a race-shaped edge case: if a request contains - only tool outputs, it is marked as requiring the live call-id state. If that - state no longer matches by worker execution time, DS4 now returns a 400 asking - for full input instead of cold-prefilling a prompt that starts with a naked - tool result. Full visible replays in thinking mode also remain marked as - reasoning-sensitive even if live state existed during parse, because the live - KV can be replaced before execution. -- Disk cache hit logs now include the key kind: - `key=responses-visible` for visible-transcript keys that restore hidden KV, - and `key=token-text` for ordinary rendered-token text keys. -- `make ds4_test && ./ds4_test --server && make ds4-server`: pass after the - stricter tool-output-only check and log-key change. -- A separate `/tmp/ds4_test_asan` build with - `-fsanitize=address,undefined` also passes `--server`. Leak detection is not - supported by Apple ASan on this platform, so the run covered address/UB - checks but not leak reporting. - -Restart / disk recovery test: - -- Stopped the old server and started a fresh `ds4-server` with the existing - `/tmp/kvcache-responses-switch3` directory. -- Resumed the existing Codex session - `019e23a9-821c-7e40-8c00-122646d7fda7` after process restart. -- Expected behavior occurred immediately: - `kv cache hit text tokens=12367 ... key=responses-visible`, followed by a - short suffix prefill and a live Responses tool-output continuation with - `ids=1`. -- After unrelated pi, curl, and opencode traffic replaced live KV, resuming - Codex again hit disk with `key=responses-visible` at the later checkpoint and - then continued the tool result live. This confirms the cross-client/session - path is not only an in-process memory shortcut. -- Shutdown persistence was also tested: stopping the server logged - `reason=shutdown key=responses-visible` for the latest Codex live state. - After a fresh restart, resuming the same Codex session hit that file: - `kv cache hit text tokens=12897 ... key=responses-visible`, then completed - the next tool call and tool-result turn normally. - -Additional live clients: - -- Pi was resumed from the saved session file and completed a real bash tool - turn. In this invocation pi sent a shorter prompt for the new turn, so DS4 - cold-prefilled that request and then accepted the tool result through - `responses live continuation match=visible-prefix ids=1`. -- opencode was run for two real shell-tool turns against the configured - `ds4/deepseek-v4-flash` provider. This path uses the OpenAI-compatible chat - API rather than `/v1/responses`, and it completed both the initial tool call - and the resumed tool call. That is useful shared-path regression coverage: - chat/completions tool checkpoint canonicalization still works while Responses - skips it. - -More protocol negatives: - -- Non-null `previous_response_id`: still HTTP 400. -- Non-null `conversation`: still HTTP 400. -- Tool output with an unknown `call_id`: still HTTP 400. -- Stateless replay of a prior tool call plus tool output, without a reasoning - item in thinking mode: HTTP 400. -- The same stateless replay shape with an explicit `reasoning` item: HTTP 200. -- The short tool-output-only happy path was re-tested after the stricter guard: - first curl request generated a `bash` function_call, second request sent only - the matching `function_call_output`, and DS4 logged - `responses live continuation match=tool-output-ids ids=1`. -- The stale-live-state failure path was also tested with real queued requests: - create a live `bash` function_call, start a long unrelated request that - replaces the live KV, then submit the old tool-output-only request while the - long request is still running. When the queued tool-output-only request - executed, DS4 returned HTTP 400 with - `Responses tool output requires live call state; replay full input instead`. - This verifies the new guard prevents cold-prefilling an orphan tool result. +Tool-result continuations use the same id and only the new output item: + +```json +{ + "previous_response_id": "resp_example", + "input": [ + { + "type": "function_call_output", + "call_id": "call_example", + "output": "tool result" + } + ] +} +``` -Post-merge stress pass, 2026-05-14: +The `call_id` must belong to a tool call generated by the referenced response. +This validates the binding without scanning a replayed assistant turn. + +`conversation` objects remain unsupported. Supplying one returns a request +error rather than silently changing continuation semantics. + +## What is retained + +After a non-error response, the server records a bounded response-state entry +keyed by its `resp_` id. The entry contains: + +- the slot, pre-generation prompt boundary, post-response token frontier, and + slot epoch for the exact in-memory KV state; +- a prompt/configuration fingerprint and the saved instructions, tool schemas, + tool choice, model syntax, and reasoning mode needed to validate a request; and +- generated function/custom tool call IDs. + +The response id is generated before either streaming or non-streaming response +serialization, so the same value is returned on the wire and used as the state +key. + +The in-memory index is FIFO-bounded and retains 4096 completed response ids by +default; `--response-state-max-ids N` changes that limit. DS4 currently has no +tenant/authentication layer, so response ids are local opaque handles rather +than authorization credentials. + +When disk KV caching is enabled, response-id persistence follows the normal +continued-cache cadence rather than synchronously copying a full checkpoint on +every completed turn. When a response crosses a continued-cache boundary, DS4 +writes its exact post-response frontier under an opaque response-id key using +the ordinary minimum-prefix and disk-budget policy. The crossed boundary is +recorded even when a write is over budget or fails, so short following turns do +not retry the same full copy. Response ids between retained boundaries remain +live-process continuations and require normal replay after a restart or slot +displacement. + +The checkpoint trailer stores the response configuration, continuation +boundary, and generated call ids. After a restart, a request for a retained id +rebuilds its bounded record directly from that KV file before parsing the delta, +then restores the exact checkpoint if no resident slot still has the frontier. +A missing, evicted, over-budget, incompatible-model, or incompatible-context +checkpoint still requires normal full replay. + +## Fast-path eligibility and fallback + +A response-id continuation is accepted only when the supplied input is an +append-only delta: `user`, `tool`, or `function` input items. It validates any +explicitly supplied prompt-affecting settings against the saved configuration. +Omitted model, instructions, tools, tool choice, and reasoning settings inherit +the saved response configuration. + +An edited branch, assistant/system history in the supplied input, or a delta +that adds hosted tool-search schemas cannot use the response-id tail path; those +complete-history forms follow the existing stateless replay path and retain its +ordinary prefix reuse. A request containing only a delta and an unknown/expired +id or changed prompt configuration cannot be reconstructed safely, so it +receives a replay-required error. A request that contains a complete replay +remains eligible for the stateless path even if its `previous_response_id` no +longer resolves. + +Before generation, queued response-id work revalidates the referenced slot and +epoch. If the live state was replaced and no compatible saved checkpoint can be +loaded, DS4 asks the client to replay instead of continuing from an unrelated +same-length frontier. + +## Rendering and KV behavior + +On a hit, DS4 does not decode, render, or tokenize the saved visible history. +It renders only the generic continuation tail, tokenizes that tail, and +synchronizes it with the saved exact frontier. This preserves hidden reasoning +and tool-call bindings from the original generation while avoiding historical +JSON/string-buffer/transient-message work. + +Stateless replay still uses exact sampled DSML tool replay where available, +then normal token-prefix, text-prefix, and disk KV matching. The older +responses-visible live-prefix path remains useful for full replay clients that +do not provide `previous_response_id`. + +## Tests and measurement + +Server unit coverage exercises response-state FIFO eviction, configuration +matching, append-only classification, saved tool-call bindings, slot/epoch +routing, response-checkpoint cadence, and a stable response id shared by +streaming and non-streaming serialization. Run the focused server tests with: + +```sh +make ds4_test && ./ds4_test --server +``` -- Merged the `responses-api` branch into `main` while preserving its branch - commits, then re-applied the exact sampled DSML tool-checkpoint gate on top of - the merged Responses live-continuation path. -- Replaced eager tool-less thinking checkpoint rebuilds with a visible-transcript - live binding. A no-tool thinking answer now remembers the visible transcript - that clients replay, while keeping the sampled hidden reasoning in KV. The - next chat/completions or Anthropic request can continue with - `cache_source: thinking-visible` if its prompt extends that visible prefix. -- Unit/build checks after the change: - `make ds4_test`, `./ds4_test --server`, and `make ds4-server` all passed. -- Live stress server: - `./ds4-server -m gguf/DeepSeek-V4-Flash-IQ2XXS-w2Q2K-AProjQ8-SExpQ8-OutQ8-chat-v2-imatrix.gguf --ctx 100000 --kv-disk-dir /tmp/ds4-kv-stress --kv-disk-space-mb 8192 --trace /tmp/ds4-response-stress-trace.txt`. -- Pi, using the OpenAI-compatible chat/completions path, completed a multi-tool - session and a resumed turn. The trace showed normal live-prefix reuse: - requests after the first cold prompt used `cache_source: memory-token` with - only short suffix prefills. -- Codex CLI, using `/v1/responses`, completed one tool-heavy turn and a resumed - turn. Responses continuations used `cache_source: responses-visible`; the - first resume after the final answer logged `ids=0` and continued from the - remembered visible prefix, then tool-result turns logged `ids=1`. -- Claude Code, using `/v1/messages`, completed real read/write/bash tool loops. - One apparent bad miss after a `Read` call was investigated in the rendered - trace: the client changed its tool schema block between requests - (`NotebookEdit` disappeared and `LSP` appeared), so the low common prefix was - a real prompt change, not a DSML replay failure. Disk recovery limited the - replay to the nearest checkpoint (`cached_tokens=22528` for a 24308-token - prompt). -- opencode, using OpenAI-compatible chat/completions, completed a tool session - and a resumed tool session. Most tool-result turns used live memory; a few - used `cache_source: memory-text` because the exact token stream and rerendered - byte prefix were text-identical but had different tokenization around a - boundary, which is the intended text-prefix fallback. -- A direct no-tool chat/completions test exercised the new thinking-visible - binding: first request logged `thinking live checkpoint remembered`, second - request logged `thinking live continuation match=visible-prefix cached=54 - prompt=66`, with no rebuild and only the new suffix prefilling. -- No trace line in this run showed `thinking checkpoint canonicalization needs - rebuild` or a tool checkpoint rebuild caused by exact sampled DSML replay. +`tests/responses_replay_bench` reports `host_ingress_ns/op` through the HTTP +endpoint, parser, worker, response-id lookup, and prefix synchronization using +a byte-tokenizer/checkpoint test fixture. Its disk-cache controls can shape a +checkpoint, verify a retained-id restart, and assert that short tails below the +next continued boundary do not write another checkpoint. It intentionally +excludes model and GPU evaluation, so it is a host-ingress regression test +rather than a model throughput result. + +A production end-to-end trace should grow one Responses conversation while +sending `previous_response_id` plus a small delta after the first turn. Record +request bytes, client-thread parse/render/tokenize CPU and allocation, +response-state hit rate, cached/prefetched tokens, and tail size separately. +Compare it with the equivalent full-visible-replay client. The intended gain is +bounded historical ingress work on response-id hits; first turns, misses, edits, +and full replays necessarily retain the existing costs. diff --git a/tests/responses_replay_bench.c b/tests/responses_replay_bench.c new file mode 100644 index 0000000000..4ee3871885 --- /dev/null +++ b/tests/responses_replay_bench.c @@ -0,0 +1,711 @@ +/* + * End-to-end /v1/responses continuation ingress benchmark. + * + * The timed operation uses a socketpair, client_main(), the resident worker, + * generate_job(), and server_session_sync(). A synthetic byte-vocabulary + * engine invokes the production rendered-chat tokenizer; its checkpoint-only + * test session preserves and appends exact token prefixes but intentionally + * does not evaluate model weights. The reported host_ingress_ns/op boundary + * is therefore CPU-side endpoint ingress rather than model or GPU latency; it + * retains HTTP dispatch, response serialization, response-id lookup, and + * saved-prefix synchronization. + * + * Before response-id support is available, each operation posts the full + * visible replay. A response-id build first posts that same initial history, + * then follows the response ids produced by real endpoint replies and posts + * only a user delta. Without the optional restart probe, the first timed + * request in both forms represents the same logical conversation state; only + * the wire form and historical parsing work differ. The restart probe adds one + * unmeasured continuation solely to validate disk restoration. + */ +#define DS4_SERVER_TEST +#define DS4_SERVER_TEST_NO_MAIN +#include "../ds4_server.c" + +#include +#include + +static volatile uint64_t responses_replay_bench_sink; +static int responses_replay_bench_diag_fd = -1; + +typedef struct { + int turns; + int iterations; + int disk_cache_mb; + int continued_interval_tokens; + int checkpoint_tokens; + uint64_t checkpoint_bytes; + bool restart_check; + bool checkpoint_expectation_set; + bool checkpoint_expect_saved; +} responses_replay_bench_config; + +typedef struct { + server srv; + pthread_t worker; + bool worker_started; +} responses_replay_bench_server; + +static void responses_replay_bench_fail(const char *message) { + int fd = responses_replay_bench_diag_fd >= 0 ? + responses_replay_bench_diag_fd : STDERR_FILENO; + dprintf(fd, "responses-replay-bench: %s\n", message); + exit(1); +} + +static double responses_replay_bench_now(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (double)ts.tv_sec + (double)ts.tv_nsec / 1000000000.0; +} + +static int responses_replay_bench_positive_int(const char *text, const char *name) { + char *end = NULL; + errno = 0; + long value = strtol(text, &end, 10); + if (errno || !text[0] || !end || *end || value <= 0 || value > INT_MAX) { + fprintf(stderr, "responses-replay-bench: invalid %s: %s\n", name, text); + exit(2); + } + return (int)value; +} + +static uint64_t responses_replay_bench_positive_u64(const char *text, + const char *name) { + char *end = NULL; + errno = 0; + unsigned long long value = strtoull(text, &end, 10); + if (errno || !text[0] || !end || *end || value == 0 || + (uint64_t)value != value) + { + fprintf(stderr, "responses-replay-bench: invalid %s: %s\n", name, text); + exit(2); + } + return (uint64_t)value; +} + +static responses_replay_bench_config responses_replay_bench_parse_options( + int argc, char **argv) { + responses_replay_bench_config cfg = { + .turns = 512, + .iterations = 100, + .continued_interval_tokens = 8192, + }; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--turns") && i + 1 < argc) { + cfg.turns = responses_replay_bench_positive_int(argv[++i], "--turns"); + } else if (!strcmp(argv[i], "--iterations") && i + 1 < argc) { + cfg.iterations = + responses_replay_bench_positive_int(argv[++i], "--iterations"); + } else if (!strcmp(argv[i], "--disk-cache-mb") && i + 1 < argc) { + cfg.disk_cache_mb = + responses_replay_bench_positive_int(argv[++i], "--disk-cache-mb"); + } else if (!strcmp(argv[i], "--continued-interval-tokens") && i + 1 < argc) { + cfg.continued_interval_tokens = responses_replay_bench_positive_int( + argv[++i], "--continued-interval-tokens"); + } else if (!strcmp(argv[i], "--checkpoint-tokens") && i + 1 < argc) { + cfg.checkpoint_tokens = responses_replay_bench_positive_int( + argv[++i], "--checkpoint-tokens"); + } else if (!strcmp(argv[i], "--checkpoint-bytes") && i + 1 < argc) { + cfg.checkpoint_bytes = responses_replay_bench_positive_u64( + argv[++i], "--checkpoint-bytes"); + } else if (!strcmp(argv[i], "--restart-check")) { + cfg.restart_check = true; + } else if (!strcmp(argv[i], "--expect-checkpoint-saved")) { + if (cfg.checkpoint_expectation_set && !cfg.checkpoint_expect_saved) { + fprintf(stderr, "responses-replay-bench: conflicting checkpoint expectations\n"); + exit(2); + } + cfg.checkpoint_expectation_set = true; + cfg.checkpoint_expect_saved = true; + } else if (!strcmp(argv[i], "--expect-checkpoint-skipped")) { + if (cfg.checkpoint_expectation_set && cfg.checkpoint_expect_saved) { + fprintf(stderr, "responses-replay-bench: conflicting checkpoint expectations\n"); + exit(2); + } + cfg.checkpoint_expectation_set = true; + cfg.checkpoint_expect_saved = false; + } else if (!strcmp(argv[i], "-h") || !strcmp(argv[i], "--help")) { + puts("usage: responses_replay_bench [--turns N] [--iterations N]"); + puts(" [--disk-cache-mb N --continued-interval-tokens N]"); + puts(" [--checkpoint-tokens N --checkpoint-bytes N --restart-check]"); + puts(" [--expect-checkpoint-saved|--expect-checkpoint-skipped]"); + exit(0); + } else { + fprintf(stderr, "responses-replay-bench: unknown option: %s\n", argv[i]); + exit(2); + } + } + if ((cfg.checkpoint_tokens || cfg.checkpoint_bytes || cfg.restart_check || + cfg.checkpoint_expectation_set) && !cfg.disk_cache_mb) + { + fprintf(stderr, "responses-replay-bench: disk checkpoint options require --disk-cache-mb\n"); + exit(2); + } + if (cfg.restart_check && cfg.checkpoint_expectation_set && + !cfg.checkpoint_expect_saved) + { + fprintf(stderr, "responses-replay-bench: restart check requires a retained checkpoint\n"); + exit(2); + } + return cfg; +} + +static char *responses_replay_bench_full_input(int turns) { + buf input = {0}; + buf_putc(&input, '['); + for (int i = 0; i < turns; i++) { + if (i) buf_putc(&input, ','); + buf_printf(&input, + "{\"type\":\"message\",\"role\":\"user\",\"content\":[{" + "\"type\":\"input_text\",\"text\":\"turn %d: explain cache reuse\"}]}," + "{\"type\":\"message\",\"role\":\"assistant\",\"content\":[{" + "\"type\":\"output_text\",\"text\":\"acknowledged %d\"}]}", + i, i); + } + buf_putc(&input, ']'); + return buf_take(&input); +} + +static void responses_replay_bench_append_delta_user(buf *input, bool *has_item, + int padding_bytes) { + if (*has_item) buf_putc(input, ','); + buf_puts(input, + "{\"type\":\"message\",\"role\":\"user\",\"content\":[{" + "\"type\":\"input_text\",\"text\":\"next question"); + for (int i = 0; i < padding_bytes; i++) buf_putc(input, 'x'); + buf_puts(input, "\"}]}"); + *has_item = true; +} + +static void responses_replay_bench_append_fixture_assistant(buf *input, bool *has_item) { + if (*has_item) buf_putc(input, ','); + buf_puts(input, + "{\"type\":\"message\",\"role\":\"assistant\",\"content\":[{" + "\"type\":\"output_text\",\"text\":\"A\"}]}"); + *has_item = true; +} + +/* The checkpoint-only session deterministically emits A for one output token. + * This builds the matching visible replay while keeping request construction + * outside the timed server round trip. */ +static char *responses_replay_bench_growing_input(const char *history, + int completed_deltas, + int initial_padding_bytes) { + size_t len = history ? strlen(history) : 0; + if (len < 2 || history[0] != '[' || history[len - 1] != ']') { + responses_replay_bench_fail("invalid full replay fixture"); + } + buf input = {0}; + buf_append(&input, history, len - 1); + bool has_item = len > 2; + for (int i = 0; i < completed_deltas; i++) { + responses_replay_bench_append_delta_user( + &input, &has_item, i == 0 ? initial_padding_bytes : 0); + responses_replay_bench_append_fixture_assistant(&input, &has_item); + } + responses_replay_bench_append_delta_user( + &input, &has_item, completed_deltas == 0 ? initial_padding_bytes : 0); + buf_putc(&input, ']'); + return buf_take(&input); +} + +static char *responses_replay_bench_request_body(const char *input, + const char *previous_response_id) { + buf body = {0}; + buf_puts(&body, + "{\"max_output_tokens\":1,\"reasoning\":{\"effort\":\"none\"},"); + if (previous_response_id && previous_response_id[0]) { + buf_puts(&body, "\"previous_response_id\":"); + json_escape(&body, previous_response_id); + buf_putc(&body, ','); + } + buf_puts(&body, "\"input\":"); + buf_puts(&body, input ? input : "[]"); + buf_putc(&body, '}'); + return buf_take(&body); +} + +static const char *responses_replay_bench_delta_input(void) { + return "[{\"type\":\"message\",\"role\":\"user\",\"content\":[{" + "\"type\":\"input_text\",\"text\":\"next question\"}]}]"; +} + +static void responses_replay_bench_write_all(int fd, const char *data, size_t len) { + while (len > 0) { + ssize_t wrote = write(fd, data, len); + if (wrote < 0 && errno == EINTR) continue; + if (wrote <= 0) responses_replay_bench_fail("failed to send HTTP request"); + data += wrote; + len -= (size_t)wrote; + } +} + +static char *responses_replay_bench_http_request(server *s, const char *body) { + int pair[2] = {-1, -1}; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, pair) != 0) { + responses_replay_bench_fail("socketpair failed"); + } + + client_arg *arg = xmalloc(sizeof(*arg)); + arg->srv = s; + arg->fd = pair[1]; + pthread_mutex_lock(&s->mu); + s->clients++; + pthread_mutex_unlock(&s->mu); + + pthread_t client; + if (pthread_create(&client, NULL, client_main, arg) != 0) { + free(arg); + pthread_mutex_lock(&s->mu); + s->clients--; + pthread_mutex_unlock(&s->mu); + close(pair[0]); + close(pair[1]); + responses_replay_bench_fail("failed to start HTTP client thread"); + } + + buf wire = {0}; + buf_printf(&wire, + "POST /v1/responses HTTP/1.1\r\n" + "Host: responses-replay-bench\r\n" + "Content-Type: application/json\r\n" + "Content-Length: %zu\r\n" + "Connection: close\r\n\r\n", + strlen(body)); + buf_puts(&wire, body); + responses_replay_bench_write_all(pair[0], wire.ptr, wire.len); + buf_free(&wire); + /* Content-Length already delimits the request. A write-side half-close is + * a client disconnect to the server, so keep this end open until the reply + * has been read. */ + + buf reply = {0}; + char chunk[4096]; + for (;;) { + ssize_t n = read(pair[0], chunk, sizeof(chunk)); + if (n < 0 && errno == EINTR) continue; + if (n < 0) { + close(pair[0]); + pthread_join(client, NULL); + responses_replay_bench_fail("failed to read HTTP response"); + } + if (n == 0) break; + buf_append(&reply, chunk, (size_t)n); + } + close(pair[0]); + pthread_join(client, NULL); + return buf_take(&reply); +} + +static char *responses_replay_bench_response_id(char *reply) { + if (!reply || !strstr(reply, " 200 ")) { + free(reply); + responses_replay_bench_fail("endpoint did not return HTTP 200"); + } + const char *body = strstr(reply, "\r\n\r\n"); + body = body ? body + 4 : reply; + const char *id = strstr(body, "\"id\":\"resp_"); + if (!id) { + free(reply); + responses_replay_bench_fail("endpoint reply did not contain a response id"); + } + id += strlen("\"id\":\""); + const char *end = strchr(id, '\"'); + if (!end) { + free(reply); + responses_replay_bench_fail("endpoint reply had a malformed response id"); + } + char *result = xstrndup(id, (size_t)(end - id)); + responses_replay_bench_sink += (uint64_t)strlen(reply) + (uint64_t)strlen(result); + free(reply); + return result; +} + +static char *responses_replay_bench_issue(server *s, const char *body) { + return responses_replay_bench_response_id( + responses_replay_bench_http_request(s, body)); +} + +static char *responses_replay_bench_issue_timed(server *s, const char *body, + double *elapsed) { + const double start = responses_replay_bench_now(); + char *reply = responses_replay_bench_http_request(s, body); + *elapsed += responses_replay_bench_now() - start; + return responses_replay_bench_response_id(reply); +} + +static void responses_replay_bench_server_init(responses_replay_bench_server *bench, + const responses_replay_bench_config *cfg, + const char *disk_dir) { + memset(bench, 0, sizeof(*bench)); + server *s = &bench->srv; + s->engine = ds4_test_engine_create_byte_tokenizer(); + if (!s->engine) responses_replay_bench_fail("failed to create tokenizer fixture"); + s->ctx_size = 262144; + s->default_tokens = 1; + s->slot_count = 1; + s->slots = calloc(1, sizeof(*s->slots)); + if (!s->slots) responses_replay_bench_fail("failed to allocate server slot"); + s->slots[0].srv = s; + s->slots[0].id = 0; +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) + s->slots[0].frontier_epoch = 1; +#endif + s->slots[0].session = + ds4_test_session_create_token_only(s->engine, s->ctx_size); + if (!s->slots[0].session) responses_replay_bench_fail("failed to create session fixture"); + if (cfg->checkpoint_bytes && + !ds4_test_session_set_payload_bytes(s->slots[0].session, + cfg->checkpoint_bytes)) + { + responses_replay_bench_fail("failed to configure checkpoint payload fixture"); + } + + pthread_mutex_init(&s->tool_mu, NULL); + pthread_mutex_init(&s->kv_mu, NULL); + pthread_mutex_init(&s->inference_mu, NULL); + pthread_mutex_init(&s->model_mu, NULL); + pthread_mutex_init(&s->mu, NULL); + pthread_mutex_init(&s->trace_mu, NULL); + pthread_cond_init(&s->model_cv, NULL); + pthread_cond_init(&s->cv, NULL); + pthread_cond_init(&s->clients_cv, NULL); + if (cfg->disk_cache_mb) { + kv_cache_options opt = kv_cache_default_options(); + /* The benchmark wants the response-id checkpoint itself, not an + * unrelated cold checkpoint of the same synthetic test session. */ + opt.cold_max_tokens = 0; + opt.continued_interval_tokens = cfg->continued_interval_tokens; + if (!disk_dir || !ds4_kvstore_open(&s->kv, disk_dir, + (uint64_t)cfg->disk_cache_mb, + false, opt, + "responses-replay-bench", NULL, NULL)) + { + responses_replay_bench_fail("failed to open disk cache fixture"); + } + } + if (pthread_create(&bench->worker, NULL, worker_main, s) != 0) { + responses_replay_bench_fail("failed to start server worker"); + } + bench->worker_started = true; +} + +static void responses_replay_bench_server_close(responses_replay_bench_server *bench) { + server *s = &bench->srv; + if (bench->worker_started) { + pthread_mutex_lock(&s->mu); + s->stopping = true; + pthread_cond_broadcast(&s->cv); + pthread_mutex_unlock(&s->mu); + pthread_join(bench->worker, NULL); + } +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) + response_state_index_free(&s->response_states); +#endif + if (s->slots) { + for (int i = 0; i < s->slot_count; i++) ds4_session_free(s->slots[i].session); + } + free(s->slots); + if (s->kv.enabled) ds4_kvstore_close(&s->kv); + pthread_mutex_destroy(&s->tool_mu); + pthread_mutex_destroy(&s->kv_mu); + pthread_mutex_destroy(&s->inference_mu); + pthread_mutex_destroy(&s->model_mu); + pthread_mutex_destroy(&s->trace_mu); + pthread_cond_destroy(&s->model_cv); + pthread_cond_destroy(&s->clients_cv); + pthread_cond_destroy(&s->cv); + pthread_mutex_destroy(&s->mu); + ds4_test_engine_free_byte_tokenizer(s->engine); + memset(bench, 0, sizeof(*bench)); +} + +static void responses_replay_bench_remove_disk_dir(const char *dir) { + if (!dir || !dir[0]) return; + DIR *d = opendir(dir); + if (d) { + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) continue; + char *path = ds4_kvstore_path_join(dir, de->d_name); + (void)unlink(path); + free(path); + } + closedir(d); + } + (void)rmdir(dir); +} + +static int responses_replay_bench_prompt_tokens(server *s, const char *body) { + request r; + char err[160] = {0}; + if (!parse_responses_request(s->engine, s, body, s->default_tokens, + s->ctx_size, &r, err, sizeof(err))) + { + dprintf(responses_replay_bench_diag_fd >= 0 ? responses_replay_bench_diag_fd : + STDERR_FILENO, + "responses-replay-bench: failed to count fixture prompt: %s\n", + err[0] ? err : "unknown error"); + exit(1); + } + const int tokens = r.prompt.len; + request_free(&r); + return tokens; +} + +static int responses_replay_bench_initial_padding(server *s, + const char *history, + int target_tokens) { + if (target_tokens == 0) return 0; + int padding = 0; + for (int attempt = 0; attempt < 3; attempt++) { + char *input = responses_replay_bench_growing_input(history, 0, padding); + char *body = responses_replay_bench_request_body(input, NULL); + const int actual = responses_replay_bench_prompt_tokens(s, body) + 1; + free(body); + free(input); + if (actual == target_tokens) return padding; + if (actual > target_tokens || target_tokens - actual > INT_MAX - padding) { + responses_replay_bench_fail("requested checkpoint token shape is smaller than the fixture"); + } + /* The byte tokenizer has no merge table, so each x appends exactly one + * rendered token. Retain the short retry for future fixture changes. */ + padding += target_tokens - actual; + } + responses_replay_bench_fail("failed to shape checkpoint token fixture"); + return 0; +} + +static uint64_t responses_replay_bench_payload_save_count(const server *s) { + return s && s->slots && s->slots[0].session ? + ds4_test_session_payload_save_count(s->slots[0].session) : 0; +} + +static int responses_replay_bench_check_initial_checkpoint( + server *s, const char *id, const responses_replay_bench_config *cfg) { +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) + response_state_entry *entry = response_state_acquire(s, id); + if (!entry) responses_replay_bench_fail("initial response state was not indexed"); + const bool saved = response_state_checkpoint_saved(s, entry); + const int live_tokens = entry->live_tokens; + response_state_release(s, entry); + if (cfg->checkpoint_tokens && live_tokens != cfg->checkpoint_tokens) { + responses_replay_bench_fail("initial response did not reach requested checkpoint token shape"); + } + if (!cfg->disk_cache_mb) return 0; + const bool expected = cfg->checkpoint_expectation_set ? + cfg->checkpoint_expect_saved : true; + if (saved != expected) { + responses_replay_bench_fail(expected ? + "expected retained response-id checkpoint was not saved" : + "oversized response-id checkpoint unexpectedly persisted"); + } + const uint64_t payload_saves = responses_replay_bench_payload_save_count(s); + if (payload_saves != (expected ? 1u : 0u)) { + responses_replay_bench_fail(expected ? + "retained response-id checkpoint staged an unexpected payload count" : + "oversized response-id checkpoint staged a payload before budget rejection"); + } + const int boundary = s->slots[0].response_state_last_checkpoint_tokens; + if (boundary <= 0) { + responses_replay_bench_fail("response-id checkpoint did not record its cadence boundary"); + } + return boundary; +#else + (void)s; + (void)id; + (void)cfg; + return 0; +#endif +} + +static void responses_replay_bench_check_tail_live_only( + server *s, const char *id, int initial_boundary, + uint64_t expected_payload_saves, + const responses_replay_bench_config *cfg) { +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) + if (!cfg->disk_cache_mb) return; + response_state_entry *entry = response_state_acquire(s, id); + if (!entry) responses_replay_bench_fail("tail response state was not indexed"); + const bool saved = response_state_checkpoint_saved(s, entry); + response_state_release(s, entry); + if (saved) { + responses_replay_bench_fail("short response unexpectedly wrote another response-id checkpoint"); + } + if (s->slots[0].response_state_last_checkpoint_tokens != initial_boundary) { + responses_replay_bench_fail("short response unexpectedly advanced response-id checkpoint cadence"); + } + if (responses_replay_bench_payload_save_count(s) != expected_payload_saves) { + responses_replay_bench_fail("short response unexpectedly staged another checkpoint payload"); + } +#else + (void)s; + (void)id; + (void)initial_boundary; + (void)expected_payload_saves; + (void)cfg; +#endif +} + +static int responses_replay_bench_quiet_stderr(void) { + fflush(stderr); + int saved = dup(STDERR_FILENO); + int nullfd = open("/dev/null", O_WRONLY); + if (saved < 0 || nullfd < 0 || dup2(nullfd, STDERR_FILENO) < 0) { + if (saved >= 0) close(saved); + if (nullfd >= 0) close(nullfd); + responses_replay_bench_fail("failed to silence endpoint logs"); + } + close(nullfd); + responses_replay_bench_diag_fd = saved; + return saved; +} + +static void responses_replay_bench_restore_stderr(int saved) { + fflush(stderr); + if (saved >= 0) { + (void)dup2(saved, STDERR_FILENO); + close(saved); + } + responses_replay_bench_diag_fd = -1; +} + +static double responses_replay_bench_full_replay(server *s, const char *history, + int initial_padding_bytes, + int iterations) { + char *warmup_input = responses_replay_bench_growing_input( + history, 0, initial_padding_bytes); + char *warmup_body = responses_replay_bench_request_body(warmup_input, NULL); + char *warmup = responses_replay_bench_issue(s, warmup_body); + free(warmup); + free(warmup_body); + free(warmup_input); + + int saved_stderr = responses_replay_bench_quiet_stderr(); + double elapsed = 0.0; + for (int i = 0; i < iterations; i++) { + char *input = responses_replay_bench_growing_input( + history, i + 1, initial_padding_bytes); + char *body = responses_replay_bench_request_body(input, NULL); + char *response_id = responses_replay_bench_issue_timed(s, body, &elapsed); + free(response_id); + free(body); + free(input); + } + responses_replay_bench_restore_stderr(saved_stderr); + return elapsed; +} + +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) && !defined(DS4_RESPONSES_REPLAY_BENCH_FORCE_FULL) +static char *responses_replay_bench_issue_initial(server *s, const char *initial_body, + const responses_replay_bench_config *cfg, + int *checkpoint_boundary, + uint64_t *payload_saves) { + char *id = responses_replay_bench_issue(s, initial_body); + if (checkpoint_boundary) { + *checkpoint_boundary = responses_replay_bench_check_initial_checkpoint(s, id, cfg); + } + if (payload_saves) { + *payload_saves = responses_replay_bench_payload_save_count(s); + } + return id; +} + +static char *responses_replay_bench_resume_after_restart( + server *s, const char *previous_id, int checkpoint_boundary, + const responses_replay_bench_config *cfg) { + const int before = ds4_session_pos(s->slots[0].session); + if (before != 0) responses_replay_bench_fail("restarted fixture retained an in-memory session"); + char *body = responses_replay_bench_request_body( + responses_replay_bench_delta_input(), previous_id); + char *next_id = responses_replay_bench_issue(s, body); + free(body); + if (ds4_session_pos(s->slots[0].session) <= before) { + free(next_id); + responses_replay_bench_fail("retained response id did not reload after restart"); + } + responses_replay_bench_check_tail_live_only( + s, next_id, checkpoint_boundary, 0, cfg); + return next_id; +} + +static double responses_replay_bench_response_id_tail( + server *s, char *previous_id, int checkpoint_boundary, + uint64_t expected_payload_saves, + const responses_replay_bench_config *cfg) { + int saved_stderr = responses_replay_bench_quiet_stderr(); + double elapsed = 0.0; + for (int i = 0; i < cfg->iterations; i++) { + char *body = responses_replay_bench_request_body( + responses_replay_bench_delta_input(), previous_id); + const int before = ds4_session_pos(s->slots[0].session); + char *next_id = responses_replay_bench_issue_timed(s, body, &elapsed); + free(body); + free(previous_id); + previous_id = next_id; + if (ds4_session_pos(s->slots[0].session) <= before) { + free(previous_id); + responses_replay_bench_restore_stderr(saved_stderr); + responses_replay_bench_fail("response-id hit did not advance the saved prefix"); + } + responses_replay_bench_check_tail_live_only( + s, previous_id, checkpoint_boundary, expected_payload_saves, cfg); + } + responses_replay_bench_restore_stderr(saved_stderr); + free(previous_id); + return elapsed; +} +#endif + +int main(int argc, char **argv) { + responses_replay_bench_config cfg = responses_replay_bench_parse_options(argc, argv); + char disk_template[] = "/tmp/ds4-responses-replay-XXXXXX"; + char *disk_dir = NULL; + if (cfg.disk_cache_mb) { + if (!mkdtemp(disk_template)) responses_replay_bench_fail("failed to create disk cache directory"); + disk_dir = xstrdup(disk_template); + } + + char *full_input = responses_replay_bench_full_input(cfg.turns); + responses_replay_bench_server bench; + responses_replay_bench_server_init(&bench, &cfg, disk_dir); + const int initial_padding = responses_replay_bench_initial_padding( + &bench.srv, full_input, cfg.checkpoint_tokens); + char *initial_input = responses_replay_bench_growing_input( + full_input, 0, initial_padding); + char *initial_body = responses_replay_bench_request_body(initial_input, NULL); + + double elapsed; +#if defined(DS4_RESPONSE_STATE_DEFAULT_MAX_IDS) && !defined(DS4_RESPONSES_REPLAY_BENCH_FORCE_FULL) + int checkpoint_boundary = 0; + uint64_t payload_saves = 0; + char *previous_id = responses_replay_bench_issue_initial( + &bench.srv, initial_body, &cfg, &checkpoint_boundary, &payload_saves); + if (cfg.restart_check) { + responses_replay_bench_server_close(&bench); + responses_replay_bench_server_init(&bench, &cfg, disk_dir); + char *resumed_id = responses_replay_bench_resume_after_restart( + &bench.srv, previous_id, checkpoint_boundary, &cfg); + free(previous_id); + previous_id = resumed_id; + payload_saves = 0; + } + elapsed = responses_replay_bench_response_id_tail( + &bench.srv, previous_id, checkpoint_boundary, payload_saves, &cfg); +#else + elapsed = responses_replay_bench_full_replay( + &bench.srv, full_input, initial_padding, cfg.iterations); +#endif + + responses_replay_bench_server_close(&bench); + responses_replay_bench_remove_disk_dir(disk_dir); + free(disk_dir); + free(initial_body); + free(initial_input); + free(full_input); + if (responses_replay_bench_sink == 0) return 1; + printf("{\"metric\":\"host_ingress_ns/op\",\"value\":%.0f}\n", + elapsed * 1000000000.0 / (double)cfg.iterations); + return 0; +}