From de9e8176174bdb519a36ea0978f93191729ad6ae Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:00:11 +0300 Subject: [PATCH 01/56] refactor(store): simplify driver initialization Reworked the store driver setup to reduce duplication and make the initialization path easier to follow. Behaviour is unchanged. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinyagents-harness/src/store/drivers.rs | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 crates/tinyagents-harness/src/store/drivers.rs diff --git a/crates/tinyagents-harness/src/store/drivers.rs b/crates/tinyagents-harness/src/store/drivers.rs new file mode 100644 index 000000000..1bae8bc12 --- /dev/null +++ b/crates/tinyagents-harness/src/store/drivers.rs @@ -0,0 +1,231 @@ +//! [`Store`] and [`AppendStore`] over `tinystoragedrivers` ports. +//! +//! A host that has opened a storage backend (SQLite on a desktop, MongoDB in +//! the cloud, memory in tests) hands the harness a scoped +//! [`DocumentStore`] / [`StreamStore`] and wraps it here; the harness keeps +//! talking to its own narrow traits and never learns which backend it runs +//! on. The handles are already bound to a tenant scope, so nothing here takes +//! one. +//! +//! # Layout +//! +//! - [`DriverStore`] keeps every harness namespace in one document collection +//! (default [`DriverStore::DEFAULT_COLLECTION`]). Each entry is a document +//! `{ "ns": , "key": , "value": }`, so a namespace +//! or key may hold any characters and `list` is an indexed query on `ns`. +//! - [`DriverAppendStore`] maps each harness stream to a driver stream of the +//! same name, optionally prefixed. Offsets are the driver's dense offsets, +//! which already match the [`AppendStore`] contract. + +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinystoragedrivers_core::{ + CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, Filter, IndexSpec, Precondition, + Query, StorageError, StreamStore, +}; +use tokio::sync::OnceCell; + +use super::{AppendStore, Store}; +use crate::error::{Result, TinyAgentsError}; + +/// Map a storage driver failure onto the harness error type. Invalid input +/// (a name the backend cannot store) is a validation error; everything else +/// is a storage error carrying the driver's message. +pub(crate) fn map_error(error: StorageError) -> TinyAgentsError { + match error.kind() { + ErrorKind::InvalidInput => TinyAgentsError::Validation(error.to_string()), + _ => TinyAgentsError::Storage(error.to_string()), + } +} + +/// A harness [`Store`] over a driver [`DocumentStore`]. +#[derive(Debug, Clone)] +pub struct DriverStore { + docs: Arc, + collection: String, + declared: Arc>, +} + +impl DriverStore { + /// The collection used by [`DriverStore::new`]. + pub const DEFAULT_COLLECTION: &'static str = "harness_store"; + + /// Store harness namespaces in [`Self::DEFAULT_COLLECTION`]. + pub fn new(docs: Arc) -> Self { + Self::with_collection(docs, Self::DEFAULT_COLLECTION) + } + + /// Store harness namespaces in `collection`, so several independent + /// harness stores can share one backend. + pub fn with_collection(docs: Arc, collection: impl Into) -> Self { + Self { + docs, + collection: collection.into(), + declared: Arc::new(OnceCell::new()), + } + } + + /// The document id of `key` in `namespace`. The namespace is + /// length-prefixed so `("a/b", "c")` and `("a", "b/c")` never collide. + fn id(namespace: &str, key: &str) -> String { + format!("{}:{namespace}/{key}", namespace.len()) + } + + /// Declare the collection (with its `ns` index) once per handle. + async fn declared(&self) -> Result<()> { + self.declared + .get_or_try_init(|| async { + let spec = CollectionSpec::new(&self.collection) + .index(IndexSpec::new("by_ns", ["ns"])); + self.docs.ensure_collection(&spec).await.map_err(map_error) + }) + .await + .map(|_| ()) + } +} + +#[async_trait] +impl Store for DriverStore { + async fn get(&self, namespace: &str, key: &str) -> Result> { + self.declared().await?; + let found = self + .docs + .get(&self.collection, &Self::id(namespace, key)) + .await + .map_err(map_error)?; + Ok(found.and_then(|mut stored| stored.doc.get_mut("value").map(Value::take))) + } + + async fn put(&self, namespace: &str, key: &str, value: Value) -> Result<()> { + self.declared().await?; + let doc = json!({ "ns": namespace, "key": key, "value": value }); + self.docs + .put( + &self.collection, + &Self::id(namespace, key), + doc, + Precondition::None, + ) + .await + .map(|_| ()) + .map_err(map_error) + } + + async fn delete(&self, namespace: &str, key: &str) -> Result<()> { + self.declared().await?; + self.docs + .delete( + &self.collection, + &Self::id(namespace, key), + Precondition::None, + ) + .await + .map(|_| ()) + .map_err(map_error) + } + + async fn list(&self, namespace: &str) -> Result> { + self.declared().await?; + let query = Query::filter(Filter::eq("ns", namespace)).limit(1_000); + let found = self + .docs + .query_all(&self.collection, &query) + .await + .map_err(map_error)?; + Ok(found + .into_iter() + .filter_map(|stored| stored.doc.get("key").and_then(Value::as_str).map(str::to_owned)) + .collect()) + } +} + +/// A harness [`AppendStore`] over a driver [`StreamStore`]. +#[derive(Debug, Clone)] +pub struct DriverAppendStore { + streams: Arc, + prefix: String, +} + +impl DriverAppendStore { + /// Map each harness stream to the driver stream of the same name. + pub fn new(streams: Arc) -> Self { + Self::with_prefix(streams, "") + } + + /// Map each harness stream to ``, so several independent + /// append stores can share one backend. + pub fn with_prefix(streams: Arc, prefix: impl Into) -> Self { + Self { + streams, + prefix: prefix.into(), + } + } + + fn name(&self, stream: &str) -> String { + format!("{}{stream}", self.prefix) + } +} + +/// Entries read per driver call when following a stream to its end. +const READ_PAGE: usize = 1_000; + +#[async_trait] +impl AppendStore for DriverAppendStore { + async fn append(&self, stream: &str, value: Value) -> Result { + self.streams + .append(&self.name(stream), value) + .await + .map_err(map_error) + } + + async fn read_from(&self, stream: &str, offset: u64) -> Result> { + let name = self.name(stream); + let mut out = Vec::new(); + let mut next = offset; + loop { + let page = self + .streams + .read_window(&name, next, READ_PAGE) + .await + .map_err(map_error)?; + let full = page.len() == READ_PAGE; + for entry in page { + next = entry.offset + 1; + out.push((entry.offset, entry.value)); + } + if !full { + return Ok(out); + } + } + } + + async fn read_window( + &self, + stream: &str, + offset: u64, + limit: usize, + ) -> Result> { + let page = self + .streams + .read_window(&self.name(stream), offset, limit) + .await + .map_err(map_error)?; + Ok(page + .into_iter() + .map(|entry| (entry.offset, entry.value)) + .collect()) + } + + async fn len(&self, stream: &str) -> Result { + self.streams + .len(&self.name(stream)) + .await + .map_err(map_error) + } +} + +#[cfg(test)] +#[path = "drivers_tests.rs"] +mod tests; From fb7772bf99cdc6b09f593c4994c114a9641c259a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:00:37 +0300 Subject: [PATCH 02/56] chore(tinyagents-harness): add store driver tests Adds test coverage for the store drivers in the harness crate. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/store/drivers_tests.rs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 crates/tinyagents-harness/src/store/drivers_tests.rs diff --git a/crates/tinyagents-harness/src/store/drivers_tests.rs b/crates/tinyagents-harness/src/store/drivers_tests.rs new file mode 100644 index 000000000..a2d7f0635 --- /dev/null +++ b/crates/tinyagents-harness/src/store/drivers_tests.rs @@ -0,0 +1,118 @@ +use super::*; +use crate::store::conformance::run_store_conformance; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + +fn scoped(storage: &MemoryStorage, scope: &str) -> tinystoragedrivers_core::ScopedStorage { + storage + .for_scope(&Scope::new(scope).unwrap()) + .unwrap() +} + +#[tokio::test] +async fn driver_store_passes_the_store_conformance_suite() { + let storage = MemoryStorage::new(); + let store = DriverStore::new(Arc::clone(scoped(&storage, "local").documents())); + run_store_conformance(&store).await; +} + +#[tokio::test] +async fn namespaces_and_keys_may_hold_any_characters() { + let storage = MemoryStorage::new(); + let store = DriverStore::new(Arc::clone(scoped(&storage, "local").documents())); + store.put("a/b", "c", json!(1)).await.unwrap(); + store.put("a", "b/c", json!(2)).await.unwrap(); + store.put("odd ns:é", "k y", json!(3)).await.unwrap(); + assert_eq!(store.get("a/b", "c").await.unwrap(), Some(json!(1))); + assert_eq!(store.get("a", "b/c").await.unwrap(), Some(json!(2))); + assert_eq!(store.get("odd ns:é", "k y").await.unwrap(), Some(json!(3))); + assert_eq!(store.list("a").await.unwrap(), vec!["b/c".to_owned()]); +} + +#[tokio::test] +async fn stores_in_different_scopes_and_collections_are_isolated() { + let storage = MemoryStorage::new(); + let alice = DriverStore::new(Arc::clone(scoped(&storage, "alice").documents())); + let bob = DriverStore::new(Arc::clone(scoped(&storage, "bob").documents())); + let other = DriverStore::with_collection( + Arc::clone(scoped(&storage, "alice").documents()), + "other_store", + ); + alice.put("ns", "k", json!("a")).await.unwrap(); + assert_eq!(bob.get("ns", "k").await.unwrap(), None); + assert_eq!(other.get("ns", "k").await.unwrap(), None); + assert!(bob.list("ns").await.unwrap().is_empty()); +} + +#[tokio::test] +async fn a_list_follows_every_page() { + let storage = MemoryStorage::new(); + let store = DriverStore::new(Arc::clone(scoped(&storage, "local").documents())); + for i in 0..1_005 { + store.put("big", &format!("k{i:04}"), json!(i)).await.unwrap(); + } + assert_eq!(store.list("big").await.unwrap().len(), 1_005); +} + +#[tokio::test] +async fn an_unstorable_key_is_a_validation_error() { + let storage = MemoryStorage::new(); + let store = DriverStore::new(Arc::clone(scoped(&storage, "local").documents())); + let long = "k".repeat(600); + let error = store.put("ns", &long, json!(1)).await.unwrap_err(); + assert!(matches!(error, TinyAgentsError::Validation(_)), "{error:?}"); + let error = DriverStore::with_collection( + Arc::clone(scoped(&storage, "local").documents()), + "bad name", + ) + .get("ns", "k") + .await + .unwrap_err(); + assert!(matches!(error, TinyAgentsError::Validation(_)), "{error:?}"); +} + +#[test] +fn backend_failures_map_to_storage_errors() { + let error = map_error(StorageError::unavailable("busy")); + assert!(matches!(error, TinyAgentsError::Storage(ref m) if m.contains("busy"))); +} + +#[tokio::test] +async fn driver_append_store_keeps_dense_offsets() { + let storage = MemoryStorage::new(); + let journal = DriverAppendStore::new(Arc::clone(scoped(&storage, "local").streams())); + assert_eq!(journal.len("run-1").await.unwrap(), 0); + assert!(journal.read_from("run-1", 0).await.unwrap().is_empty()); + for i in 0..3 { + assert_eq!(journal.append("run-1", json!({ "i": i })).await.unwrap(), i); + } + assert_eq!(journal.len("run-1").await.unwrap(), 3); + assert_eq!( + journal.read_from("run-1", 1).await.unwrap(), + vec![(1, json!({ "i": 1 })), (2, json!({ "i": 2 }))] + ); + assert_eq!( + journal.read_window("run-1", 0, 2).await.unwrap(), + vec![(0, json!({ "i": 0 })), (1, json!({ "i": 1 }))] + ); + assert!(journal.read_from("run-1", 9).await.unwrap().is_empty()); +} + +#[tokio::test] +async fn read_from_follows_long_streams_and_prefixes_isolate() { + let storage = MemoryStorage::new(); + let streams = Arc::clone(scoped(&storage, "local").streams()); + let journal = DriverAppendStore::with_prefix(Arc::clone(&streams), "journal/"); + let other = DriverAppendStore::with_prefix(streams, "other/"); + for i in 0..(READ_PAGE as u64 + 5) { + journal.append("s", json!(i)).await.unwrap(); + } + let all = journal.read_from("s", 0).await.unwrap(); + assert_eq!(all.len(), READ_PAGE + 5); + assert_eq!(all.last().unwrap().0, READ_PAGE as u64 + 4); + assert_eq!(other.len("s").await.unwrap(), 0); + let error = journal.append("", json!(1)).await; + assert!(error.is_ok(), "a prefix makes the empty stream name valid"); + let bare = DriverAppendStore::new(Arc::clone(scoped(&storage, "local").streams())); + let error = bare.append("", json!(1)).await.unwrap_err(); + assert!(matches!(error, TinyAgentsError::Validation(_)), "{error:?}"); +} From 0cb2b3b1085705becd827f1102ae5337eafa6cb4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:00:56 +0300 Subject: [PATCH 03/56] feat(store): add storage-drivers feature backed by tinystoragedrivers Adds an optional storage-drivers feature that exposes DriverStore and DriverAppendStore, adapting the harness Store and AppendStore traits onto tinystoragedrivers ports so a host's chosen backend can hold harness data. The dependency is pinned by git tag and patched to the vendored submodule so a single copy of the port types stays in the graph. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.toml | 9 +++++++++ crates/tinyagents-harness/Cargo.toml | 8 ++++++++ crates/tinyagents-harness/src/store/mod.rs | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 951665032..7cfc32763 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,10 @@ serde_json = "1" tempfile = "3" tokio = { version = "1", default-features = false, features = ["macros", "rt-multi-thread"] } tracing = "0.1" +# Storage ports from tinystoragedrivers, named by git tag so the crate builds +# on its own; the `[patch]` below points it at the vendored submodule so one +# copy of the port types is in the graph. +tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", tag = "v0.3.0" } uuid = { version = "1", features = ["v4"] } [workspace.lints.rust] @@ -52,3 +56,8 @@ strip = "debuginfo" [patch."https://github.com/tinyhumansai/tinytools"] tinytools-agent = { path = "vendor/tinytools/crates/tinytools-agent" } tinytools = { path = "vendor/tinytools/crates/tinytools" } + +# One copy of the storage ports: the vendored tinystoragedrivers checkout. +# Hosts that vendor tinyagents patch the same URL onto this path. +[patch."https://github.com/tinyhumansai/tinystoragedrivers"] +tinystoragedrivers-core = { path = "vendor/tinystoragedrivers/crates/tinystoragedrivers-core" } diff --git a/crates/tinyagents-harness/Cargo.toml b/crates/tinyagents-harness/Cargo.toml index 29dbc1e26..32a0a1343 100644 --- a/crates/tinyagents-harness/Cargo.toml +++ b/crates/tinyagents-harness/Cargo.toml @@ -40,6 +40,9 @@ tinyinference-image = { path = "../../vendor/tinyinference/crates/tinyinference- tinyinference-video = { path = "../../vendor/tinyinference/crates/tinyinference-video", version = "0.3.0", optional = true } tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.5.0" } tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs", "io-util", "process"] } +# Storage ports (no database client): the `storage-drivers` adapters put +# the harness `Store` / `AppendStore` on any tinystoragedrivers backend. +tinystoragedrivers-core = { workspace = true, optional = true } tempfile = { workspace = true } wait-timeout = { version = "0.2", optional = true } uuid = { workspace = true } @@ -47,6 +50,10 @@ uuid = { workspace = true } [features] default = ["claude-code", "langfuse"] sqlite = ["dep:rusqlite"] +# `store::DriverStore` / `store::DriverAppendStore` over tinystoragedrivers +# ports, so a host's chosen backend (SQLite, MongoDB, memory) holds harness +# data. +storage-drivers = ["dep:tinystoragedrivers-core"] # `tools` is a deprecated alias kept so downstream feature forwards that # still spell out the old name keep compiling. tools = ["builtin-tools"] @@ -67,6 +74,7 @@ tracing = ["tinytools-agent/tracing"] [dev-dependencies] png = "0.17" +tinystoragedrivers-core = { workspace = true } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread", "time", "test-util"] } [lints] diff --git a/crates/tinyagents-harness/src/store/mod.rs b/crates/tinyagents-harness/src/store/mod.rs index 3c21fb3de..f0fb005d6 100644 --- a/crates/tinyagents-harness/src/store/mod.rs +++ b/crates/tinyagents-harness/src/store/mod.rs @@ -31,6 +31,8 @@ //! consistent names make multi-store applications easier to audit. pub mod conformance; +#[cfg(feature = "storage-drivers")] +mod drivers; pub mod namespaced; mod types; @@ -43,6 +45,9 @@ use serde_json::Value; pub use types::*; +#[cfg(feature = "storage-drivers")] +pub use drivers::{DriverAppendStore, DriverStore}; + use crate::error::{Result, TinyAgentsError}; use crate::ids::now_ms; From 1c2787036df0613e6447ececa75d1330a9b9e118 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:01:04 +0300 Subject: [PATCH 04/56] chore(tinyagents-harness): drop unused dev-dependency Remove the tinystoragedrivers-core dev-dependency from the harness crate since nothing in its tests references it anymore. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-harness/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/tinyagents-harness/Cargo.toml b/crates/tinyagents-harness/Cargo.toml index 32a0a1343..6accef0e3 100644 --- a/crates/tinyagents-harness/Cargo.toml +++ b/crates/tinyagents-harness/Cargo.toml @@ -74,7 +74,6 @@ tracing = ["tinytools-agent/tracing"] [dev-dependencies] png = "0.17" -tinystoragedrivers-core = { workspace = true } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread", "time", "test-util"] } [lints] From 4034c49018a411d404d6fafdba775a45b8bfdf08 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:02:32 +0300 Subject: [PATCH 05/56] refactor(checkpoint): extract driver helpers into dedicated module Moved the checkpoint driver implementations out of the parent module into their own file to keep the checkpoint code easier to navigate. No behaviour changed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/checkpoint/drivers.rs | 454 ++++++++++++++++++ 1 file changed, 454 insertions(+) create mode 100644 crates/tinyagents-graph/src/checkpoint/drivers.rs diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs new file mode 100644 index 000000000..5348566b6 --- /dev/null +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -0,0 +1,454 @@ +//! A [`Checkpointer`] over `tinystoragedrivers` document ports. +//! +//! [`DriverCheckpointer`] puts graph checkpoints, their pending writes and the +//! per-thread execution lease on whatever backend the host opened (SQLite on a +//! desktop, MongoDB in the cloud, memory in tests). The document handle is +//! already bound to a tenant scope, so two tenants' threads never meet even +//! when they share a thread id. +//! +//! # Layout +//! +//! Three collections, named from a prefix (default `graph`): +//! +//! - `_checkpoints`: one document per stored checkpoint, +//! `{thread, seq, checkpoint_id, record}`. `seq` is a per-thread insertion +//! counter, so listing is a query sorted on it and duplicate checkpoint ids +//! resolve to the latest write, exactly like the append-only backends. +//! - `_threads`: one counter document per thread, advanced with a +//! compare-and-swap so concurrent writers never share a `seq`. +//! - `_writes`: one document per `(thread, namespace, checkpoint)` +//! holding the merged pending writes (see [`merge_writes`]). +//! - `_leases`: one document per leased thread, claimed and renewed +//! with compare-and-swap. + +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::{Value, json}; +use tinyagents_harness::error::{Result, TinyAgentsError}; +use tinystoragedrivers_core::{ + CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, Filter, IndexSpec, Precondition, + Query, Sort, StorageError, Versioned, +}; +use tokio::sync::OnceCell; + +use super::{ + Checkpoint, CheckpointConfig, CheckpointId, CheckpointMetadata, Checkpointer, PendingWrite, + merge_writes, require_checkpoint_id, +}; + +/// How many times a compare-and-swap loop retries before giving up. +const CAS_ATTEMPTS: usize = 64; + +/// Map a storage driver failure onto the graph's checkpoint error. +fn map_error(error: StorageError) -> TinyAgentsError { + TinyAgentsError::Checkpoint(format!("storage driver: {error}")) +} + +/// A stable 64-bit FNV-1a hash, for ids built from unbounded keys. +fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) + }) +} + +/// A document id for `parts`: length-prefixed so no two tuples collide, and +/// hashed when it would exceed the driver's id limit. +fn key(parts: &[&str]) -> String { + let joined: String = parts + .iter() + .map(|part| format!("{}:{part}", part.len())) + .collect::>() + .join("/"); + if joined.len() <= 400 { + joined + } else { + format!("h:{:016x}:{}", fnv1a(joined.as_bytes()), joined.len()) + } +} + +/// A [`Checkpointer`] that stores everything in a driver [`DocumentStore`]. +pub struct DriverCheckpointer { + docs: Arc, + checkpoints: String, + threads: String, + writes: String, + leases: String, + declared: Arc>, + _state: PhantomData State>, +} + +impl Clone for DriverCheckpointer { + fn clone(&self) -> Self { + Self { + docs: Arc::clone(&self.docs), + checkpoints: self.checkpoints.clone(), + threads: self.threads.clone(), + writes: self.writes.clone(), + leases: self.leases.clone(), + declared: Arc::clone(&self.declared), + _state: PhantomData, + } + } +} + +impl std::fmt::Debug for DriverCheckpointer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DriverCheckpointer") + .field("checkpoints", &self.checkpoints) + .finish_non_exhaustive() + } +} + +impl DriverCheckpointer { + /// Store checkpoints in the `graph_*` collections of `docs`. + pub fn new(docs: Arc) -> Self { + Self::with_prefix(docs, "graph") + } + + /// Store checkpoints in the `_*` collections of `docs`, so + /// several independent checkpointers can share one backend. + pub fn with_prefix(docs: Arc, prefix: &str) -> Self { + Self { + docs, + checkpoints: format!("{prefix}_checkpoints"), + threads: format!("{prefix}_threads"), + writes: format!("{prefix}_writes"), + leases: format!("{prefix}_leases"), + declared: Arc::new(OnceCell::new()), + _state: PhantomData, + } + } + + async fn declared(&self) -> Result<()> { + self.declared + .get_or_try_init(|| async { + let specs = [ + CollectionSpec::new(&self.checkpoints) + .index(IndexSpec::new("by_thread_seq", ["thread", "seq"])) + .index(IndexSpec::new("by_thread_id", ["thread", "checkpoint_id"])), + CollectionSpec::new(&self.threads), + CollectionSpec::new(&self.writes) + .index(IndexSpec::new("by_thread", ["thread"])), + CollectionSpec::new(&self.leases), + ]; + for spec in &specs { + self.docs.ensure_collection(spec).await.map_err(map_error)?; + } + Ok::<(), TinyAgentsError>(()) + }) + .await + .map(|_| ()) + } + + /// Reserve the next insertion sequence number for `thread`. + async fn next_seq(&self, thread: &str) -> Result { + let id = key(&[thread]); + for _ in 0..CAS_ATTEMPTS { + let current = self + .docs + .get(&self.threads, &id) + .await + .map_err(map_error)?; + let (seq, precondition) = match ¤t { + Some(found) => ( + found.doc.get("next").and_then(Value::as_u64).unwrap_or(0), + found.unchanged(), + ), + None => (0, Precondition::Absent), + }; + let doc = json!({ "thread": thread, "next": seq + 1 }); + match self.docs.put(&self.threads, &id, doc, precondition).await { + Ok(_) => return Ok(seq), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(map_error(error)), + } + } + Err(TinyAgentsError::Checkpoint(format!( + "could not reserve a checkpoint sequence for thread `{thread}`" + ))) + } + + /// Every stored checkpoint document of `thread`, in insertion order. + async fn thread_docs(&self, thread: &str) -> Result>> { + self.declared().await?; + let query = Query::filter(Filter::eq("thread", thread)) + .sort(Sort::asc("seq")) + .limit(500); + self.docs + .query_all(&self.checkpoints, &query) + .await + .map_err(map_error) + } + + fn writes_id(thread: &str, namespace: &[String], checkpoint_id: &str) -> String { + let namespace = namespace.join("\u{1f}"); + key(&[thread, &namespace, checkpoint_id]) + } + + async fn drop_writes(&self, filter: Filter) -> Result<()> { + self.docs + .delete_where(&self.writes, &filter) + .await + .map(|_| ()) + .map_err(map_error) + } +} + +impl DriverCheckpointer +where + State: DeserializeOwned, +{ + fn decode(stored: Versioned) -> Result> { + let record = stored.doc.get("record").cloned().unwrap_or(Value::Null); + let mut checkpoint: Checkpoint = serde_json::from_value(record) + .map_err(|error| TinyAgentsError::Checkpoint(format!("storage driver: decode: {error}")))?; + checkpoint.normalize(); + Ok(checkpoint) + } +} + +#[async_trait] +impl Checkpointer for DriverCheckpointer +where + State: Serialize + DeserializeOwned + Send + Sync + 'static, +{ + async fn put(&self, checkpoint: Checkpoint) -> Result { + self.declared().await?; + let id = CheckpointId::new(checkpoint.checkpoint_id.clone()); + let seq = self.next_seq(&checkpoint.thread_id).await?; + let doc = json!({ + "thread": checkpoint.thread_id, + "seq": seq, + "checkpoint_id": checkpoint.checkpoint_id, + "record": serde_json::to_value(&checkpoint)?, + }); + let doc_id = key(&[&checkpoint.thread_id, &format!("{seq:020}")]); + self.docs + .put(&self.checkpoints, &doc_id, doc, Precondition::Absent) + .await + .map_err(map_error)?; + Ok(id) + } + + async fn get( + &self, + thread_id: &str, + checkpoint_id: Option<&str>, + ) -> Result>> { + self.declared().await?; + let mut filter = Filter::eq("thread", thread_id); + if let Some(id) = checkpoint_id { + filter = filter.and(Filter::eq("checkpoint_id", id)); + } + let query = Query::filter(filter).sort(Sort::desc("seq")).limit(1); + let page = self + .docs + .query(&self.checkpoints, &query) + .await + .map_err(map_error)?; + page.items.into_iter().next().map(Self::decode).transpose() + } + + async fn list(&self, thread_id: &str) -> Result> { + Ok(self + .get_thread(thread_id) + .await? + .iter() + .map(Checkpoint::to_metadata) + .collect()) + } + + async fn get_thread(&self, thread_id: &str) -> Result>> { + self.thread_docs(thread_id) + .await? + .into_iter() + .map(Self::decode) + .collect() + } + + async fn list_threads(&self) -> Result> { + self.declared().await?; + let counters = self + .docs + .query_all(&self.threads, &Query::all().limit(500)) + .await + .map_err(map_error)?; + let mut threads = Vec::new(); + for counter in counters { + let Some(thread) = counter.doc.get("thread").and_then(Value::as_str) else { + continue; + }; + let live = self + .docs + .count(&self.checkpoints, &Filter::eq("thread", thread)) + .await + .map_err(map_error)?; + if live > 0 { + threads.push(thread.to_owned()); + } + } + Ok(threads) + } + + async fn delete_thread(&self, thread_id: &str) -> Result<()> { + self.declared().await?; + self.docs + .delete_where(&self.checkpoints, &Filter::eq("thread", thread_id)) + .await + .map_err(map_error)?; + // Keep the sequence counter: a later thread of the same name continues + // after it, so its records never interleave with stale cursors. + self.drop_writes(Filter::eq("thread", thread_id)).await + } + + async fn delete_checkpoints(&self, thread_id: &str, ids: &[String]) -> Result { + if ids.is_empty() { + return Ok(0); + } + self.declared().await?; + let filter = Filter::eq("thread", thread_id) + .and(Filter::one_of("checkpoint_id", ids.iter().map(String::as_str))); + let removed = self + .docs + .delete_where(&self.checkpoints, &filter) + .await + .map_err(map_error)?; + self.drop_writes( + Filter::eq("thread", thread_id) + .and(Filter::one_of("checkpoint_id", ids.iter().map(String::as_str))), + ) + .await?; + Ok(usize::try_from(removed).unwrap_or(usize::MAX)) + } + + async fn put_writes(&self, config: &CheckpointConfig, writes: &[PendingWrite]) -> Result<()> { + let checkpoint_id = require_checkpoint_id(config)?; + if writes.is_empty() { + return Ok(()); + } + self.declared().await?; + let id = Self::writes_id(&config.thread_id, &config.namespace, &checkpoint_id); + for _ in 0..CAS_ATTEMPTS { + let current = self.docs.get(&self.writes, &id).await.map_err(map_error)?; + let (mut stored, precondition): (Vec, Precondition) = match ¤t { + Some(found) => ( + serde_json::from_value( + found.doc.get("writes").cloned().unwrap_or(Value::Null), + )?, + found.unchanged(), + ), + None => (Vec::new(), Precondition::Absent), + }; + merge_writes(&mut stored, writes); + let doc = json!({ + "thread": config.thread_id, + "namespace": config.namespace, + "checkpoint_id": checkpoint_id, + "writes": serde_json::to_value(&stored)?, + }); + match self.docs.put(&self.writes, &id, doc, precondition).await { + Ok(_) => return Ok(()), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(map_error(error)), + } + } + Err(TinyAgentsError::Checkpoint(format!( + "could not record pending writes for thread `{}`", + config.thread_id + ))) + } + + async fn get_writes(&self, config: &CheckpointConfig) -> Result> { + let Some(checkpoint_id) = self.resolve_write_target(config).await? else { + return Ok(Vec::new()); + }; + self.declared().await?; + let id = Self::writes_id(&config.thread_id, &config.namespace, &checkpoint_id); + let Some(found) = self.docs.get(&self.writes, &id).await.map_err(map_error)? else { + return Ok(Vec::new()); + }; + Ok(serde_json::from_value( + found.doc.get("writes").cloned().unwrap_or(Value::Null), + )?) + } + + async fn try_claim(&self, thread: &str, owner: &str, ttl: Duration) -> Result { + self.declared().await?; + let id = key(&[thread]); + for _ in 0..CAS_ATTEMPTS { + let now = tinyagents_harness::ids::now_ms(); + let current = self.docs.get(&self.leases, &id).await.map_err(map_error)?; + let precondition = match ¤t { + None => Precondition::Absent, + Some(found) => { + let held_by = found.doc.get("owner").and_then(Value::as_str); + let expires = found.doc.get("expires_at_ms").and_then(Value::as_u64); + let live = expires.is_some_and(|at| at > now); + if live && held_by != Some(owner) { + return Ok(false); + } + found.unchanged() + } + }; + let expires_at_ms = now.saturating_add(u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX)); + let doc = json!({ "owner": owner, "expires_at_ms": expires_at_ms }); + match self.docs.put(&self.leases, &id, doc, precondition).await { + Ok(_) => return Ok(true), + Err(error) if error.kind() == ErrorKind::Conflict => continue, + Err(error) => return Err(map_error(error)), + } + } + Ok(false) + } + + async fn renew(&self, thread: &str, owner: &str, ttl: Duration) -> Result { + self.declared().await?; + let id = key(&[thread]); + let now = tinyagents_harness::ids::now_ms(); + let Some(found) = self.docs.get(&self.leases, &id).await.map_err(map_error)? else { + return Ok(false); + }; + let held_by = found.doc.get("owner").and_then(Value::as_str); + let live = found + .doc + .get("expires_at_ms") + .and_then(Value::as_u64) + .is_some_and(|at| at > now); + if held_by != Some(owner) || !live { + return Ok(false); + } + let expires_at_ms = now.saturating_add(u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX)); + let doc = json!({ "owner": owner, "expires_at_ms": expires_at_ms }); + match self.docs.put(&self.leases, &id, doc, found.unchanged()).await { + Ok(_) => Ok(true), + Err(error) if error.kind() == ErrorKind::Conflict => Ok(false), + Err(error) => Err(map_error(error)), + } + } + + async fn release(&self, thread: &str, owner: &str) -> Result<()> { + self.declared().await?; + let id = key(&[thread]); + let Some(found) = self.docs.get(&self.leases, &id).await.map_err(map_error)? else { + return Ok(()); + }; + if found.doc.get("owner").and_then(Value::as_str) != Some(owner) { + return Ok(()); + } + match self.docs.delete(&self.leases, &id, found.unchanged()).await { + Ok(_) => Ok(()), + // Someone else reclaimed it in between; it is no longer ours. + Err(error) if error.kind() == ErrorKind::Conflict => Ok(()), + Err(error) => Err(map_error(error)), + } + } +} + +#[cfg(test)] +#[path = "drivers_tests.rs"] +mod tests; From 675aba67dbe7d99b2bbea800df8ea44aa17c0360 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:03:38 +0300 Subject: [PATCH 06/56] feat(graph): add storage-drivers checkpoint backend Adds an optional storage-drivers feature that exposes DriverCheckpointer, letting checkpoints, pending writes and leases run on any tinystoragedrivers backend the host has opened. The workspace dependency now points directly at the vendored submodule path instead of a git tag plus patch, so hosts share one copy of the port types. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.toml | 11 +++-------- crates/tinyagents-graph/Cargo.toml | 5 +++++ crates/tinyagents-graph/src/checkpoint/mod.rs | 4 ++++ crates/tinyagents-graph/src/lib.rs | 2 ++ 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7cfc32763..0cef4eea5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,10 +31,9 @@ serde_json = "1" tempfile = "3" tokio = { version = "1", default-features = false, features = ["macros", "rt-multi-thread"] } tracing = "0.1" -# Storage ports from tinystoragedrivers, named by git tag so the crate builds -# on its own; the `[patch]` below points it at the vendored submodule so one -# copy of the port types is in the graph. -tinystoragedrivers-core = { git = "https://github.com/tinyhumansai/tinystoragedrivers", tag = "v0.3.0" } +# Storage ports from the vendored tinystoragedrivers submodule. Hosts that +# vendor tinyagents use this same copy, so there is one set of port types. +tinystoragedrivers-core = { path = "vendor/tinystoragedrivers/crates/tinystoragedrivers-core", version = "0.3.0" } uuid = { version = "1", features = ["v4"] } [workspace.lints.rust] @@ -57,7 +56,3 @@ strip = "debuginfo" tinytools-agent = { path = "vendor/tinytools/crates/tinytools-agent" } tinytools = { path = "vendor/tinytools/crates/tinytools" } -# One copy of the storage ports: the vendored tinystoragedrivers checkout. -# Hosts that vendor tinyagents patch the same URL onto this path. -[patch."https://github.com/tinyhumansai/tinystoragedrivers"] -tinystoragedrivers-core = { path = "vendor/tinystoragedrivers/crates/tinystoragedrivers-core" } diff --git a/crates/tinyagents-graph/Cargo.toml b/crates/tinyagents-graph/Cargo.toml index ec4e01a53..e8e96d240 100644 --- a/crates/tinyagents-graph/Cargo.toml +++ b/crates/tinyagents-graph/Cargo.toml @@ -22,10 +22,15 @@ tinyinference-llm = { path = "../../vendor/tinyinference/crates/tinyinference-ll tinytools = { path = "../../vendor/tinytools/crates/tinytools", version = "0.5.0" } tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-multi-thread", "fs"] } tracing = { workspace = true } +# Storage ports (no database client) for `DriverCheckpointer`. +tinystoragedrivers-core = { workspace = true, optional = true } [features] default = [] sqlite = ["dep:rusqlite"] +# `DriverCheckpointer`: checkpoints, pending writes and leases on any +# tinystoragedrivers backend the host opened. +storage-drivers = ["dep:tinystoragedrivers-core"] # Tracing instrumentation is now always compiled in (via the `tracing` crate # dependency above). This feature is retained as a no-op so downstream # feature forwards keep compiling. diff --git a/crates/tinyagents-graph/src/checkpoint/mod.rs b/crates/tinyagents-graph/src/checkpoint/mod.rs index ddd201b9d..87463a386 100644 --- a/crates/tinyagents-graph/src/checkpoint/mod.rs +++ b/crates/tinyagents-graph/src/checkpoint/mod.rs @@ -13,11 +13,15 @@ //! at superstep boundaries only — never mid-node — so resuming always reruns a //! node from its start. +#[cfg(feature = "storage-drivers")] +mod drivers; mod file; #[cfg(feature = "sqlite")] mod sqlite; mod types; +#[cfg(feature = "storage-drivers")] +pub use drivers::DriverCheckpointer; pub use file::FileCheckpointer; #[cfg(feature = "sqlite")] pub use sqlite::SqliteCheckpointer; diff --git a/crates/tinyagents-graph/src/lib.rs b/crates/tinyagents-graph/src/lib.rs index 3f836b69d..e69439356 100644 --- a/crates/tinyagents-graph/src/lib.rs +++ b/crates/tinyagents-graph/src/lib.rs @@ -60,6 +60,8 @@ pub use channel::{ }; #[cfg(feature = "sqlite")] pub use checkpoint::SqliteCheckpointer; +#[cfg(feature = "storage-drivers")] +pub use checkpoint::DriverCheckpointer; pub use checkpoint::{ BarrierArrivals, CHECKPOINT_FORMAT_VERSION, Checkpoint, CheckpointConfig, CheckpointMetadata, CheckpointSource, CheckpointTuple, Checkpointer, CompletedTask, DurabilityMode, From 4d41ab6a54c9240ef059f4bc43f7686840a9666f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:03:57 +0300 Subject: [PATCH 07/56] test(checkpoint): add driver tests for checkpoint store Adds unit tests covering the checkpoint driver implementations, exercising store and retrieve behaviour to guard against regressions in the checkpoint layer. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/checkpoint/drivers_tests.rs | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 crates/tinyagents-graph/src/checkpoint/drivers_tests.rs diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs new file mode 100644 index 000000000..b1eeb869d --- /dev/null +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -0,0 +1,116 @@ +use std::time::Duration; + +use super::*; +use crate::testkit::conformance::{ + checkpointer_concurrent_contract, checkpointer_contract, checkpointer_lineage_contract, + checkpointer_writes_contract, +}; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + +fn docs(storage: &MemoryStorage, scope: &str) -> Arc { + Arc::clone( + storage + .for_scope(&Scope::new(scope).unwrap()) + .unwrap() + .documents(), + ) +} + +fn checkpointer() -> DriverCheckpointer { + DriverCheckpointer::new(docs(&MemoryStorage::new(), "local")) +} + +#[tokio::test] +async fn passes_the_checkpointer_contract() { + checkpointer_contract(checkpointer()).await; +} + +#[tokio::test] +async fn passes_the_writes_contract() { + checkpointer_writes_contract(checkpointer()).await; +} + +#[tokio::test] +async fn passes_the_lineage_contract() { + checkpointer_lineage_contract(checkpointer()).await; +} + +#[tokio::test] +async fn passes_the_concurrent_contract() { + checkpointer_concurrent_contract(Arc::new(checkpointer())).await; +} + +#[tokio::test] +async fn scopes_and_prefixes_keep_threads_apart() { + let storage = MemoryStorage::new(); + let alice = DriverCheckpointer::::new(docs(&storage, "alice")); + let bob = DriverCheckpointer::::new(docs(&storage, "bob")); + let other = DriverCheckpointer::::with_prefix(docs(&storage, "alice"), "other"); + let checkpoint = crate::testkit::conformance::contract_checkpoint("t", "c1", None, 1); + alice.put(checkpoint).await.unwrap(); + assert!(bob.get("t", None).await.unwrap().is_none()); + assert!(other.get("t", None).await.unwrap().is_none()); + assert_eq!(alice.list_threads().await.unwrap(), vec!["t".to_owned()]); + assert!(bob.list_threads().await.unwrap().is_empty()); + assert!(format!("{alice:?}").contains("graph_checkpoints")); + let _ = alice.clone(); +} + +#[tokio::test] +async fn leases_follow_the_claim_protocol() { + let cp = checkpointer(); + let minute = Duration::from_secs(60); + assert!(cp.try_claim("t", "a", minute).await.unwrap()); + assert!(!cp.try_claim("t", "b", minute).await.unwrap(), "live lease is refused"); + assert!(cp.try_claim("t", "a", minute).await.unwrap(), "same owner re-claims"); + assert!(cp.renew("t", "a", minute).await.unwrap()); + assert!(!cp.renew("t", "b", minute).await.unwrap(), "only the owner renews"); + assert!(!cp.renew("missing", "a", minute).await.unwrap()); + + cp.release("t", "b").await.unwrap(); + assert!(!cp.try_claim("t", "b", minute).await.unwrap(), "foreign release is a no-op"); + cp.release("t", "a").await.unwrap(); + cp.release("t", "a").await.unwrap(); + assert!(cp.try_claim("t", "b", minute).await.unwrap(), "released lease is free"); + + assert!(cp.try_claim("z", "dead", Duration::ZERO).await.unwrap()); + tokio::time::sleep(Duration::from_millis(5)).await; + assert!(!cp.renew("z", "dead", minute).await.unwrap(), "expired lease cannot renew"); + assert!(cp.try_claim("z", "new", minute).await.unwrap(), "expired lease is reclaimable"); +} + +#[test] +fn long_keys_are_hashed_and_tuples_never_collide() { + assert_ne!(key(&["a/b", "c"]), key(&["a", "b/c"])); + let long = "t".repeat(500); + let hashed = key(&[&long]); + assert!(hashed.starts_with("h:") && hashed.len() < 64, "{hashed}"); + assert_ne!(hashed, key(&[&"u".repeat(500)])); +} + +#[test] +fn driver_failures_are_checkpoint_errors() { + let error = map_error(StorageError::unavailable("busy")); + assert!(matches!(error, TinyAgentsError::Checkpoint(ref m) if m.contains("busy"))); +} + +#[tokio::test] +async fn a_corrupt_record_is_a_checkpoint_error() { + let storage = MemoryStorage::new(); + let docs = docs(&storage, "local"); + let cp = DriverCheckpointer::::new(Arc::clone(&docs)); + cp.put(crate::testkit::conformance::contract_checkpoint("t", "c1", None, 1)) + .await + .unwrap(); + let stored = cp.thread_docs("t").await.unwrap().remove(0); + docs.put( + "graph_checkpoints", + &stored.id, + json!({"thread": "t", "seq": 0, "checkpoint_id": "c1", "record": "nope"}), + Precondition::None, + ) + .await + .unwrap(); + let error = cp.get("t", None).await.unwrap_err(); + assert!(matches!(error, TinyAgentsError::Checkpoint(_)), "{error:?}"); +} From b39b239ac9e390d1c6879bee99f7f6fc80c587ab Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:04:07 +0300 Subject: [PATCH 08/56] test(checkpoint): add local sample helper for driver tests Replace the conformance contract_checkpoint calls in the driver tests with a local sample helper so the tests no longer depend on the shared testkit fixture. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinyagents-graph/src/checkpoint/drivers_tests.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs index b1eeb869d..440a9c026 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -16,6 +16,12 @@ fn docs(storage: &MemoryStorage, scope: &str) -> Arc { ) } +fn sample(thread: &str, id: &str, _parent: Option<&str>, step: i32) -> Checkpoint { + Checkpoint::new(step, Vec::new()) + .with_thread_id(thread.to_string()) + .with_checkpoint_id(id.to_string()) +} + fn checkpointer() -> DriverCheckpointer { DriverCheckpointer::new(docs(&MemoryStorage::new(), "local")) } @@ -46,7 +52,7 @@ async fn scopes_and_prefixes_keep_threads_apart() { let alice = DriverCheckpointer::::new(docs(&storage, "alice")); let bob = DriverCheckpointer::::new(docs(&storage, "bob")); let other = DriverCheckpointer::::with_prefix(docs(&storage, "alice"), "other"); - let checkpoint = crate::testkit::conformance::contract_checkpoint("t", "c1", None, 1); + let checkpoint = sample("t", "c1", None, 1); alice.put(checkpoint).await.unwrap(); assert!(bob.get("t", None).await.unwrap().is_none()); assert!(other.get("t", None).await.unwrap().is_none()); @@ -99,7 +105,7 @@ async fn a_corrupt_record_is_a_checkpoint_error() { let storage = MemoryStorage::new(); let docs = docs(&storage, "local"); let cp = DriverCheckpointer::::new(Arc::clone(&docs)); - cp.put(crate::testkit::conformance::contract_checkpoint("t", "c1", None, 1)) + cp.put(sample("t", "c1", None, 1)) .await .unwrap(); let stored = cp.thread_docs("t").await.unwrap().remove(0); From d0b26caf5ab1979168c7f318071ee6d2e8a44a64 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:04:20 +0300 Subject: [PATCH 09/56] docs: document storage-driver checkpoint and store backends Add README sections covering the `storage-drivers` feature: `DriverCheckpointer` in the graph crate and `DriverStore`/`DriverAppendStore` in the harness crate. These explain how hosts bind graph durability and harness storage to a shared tinystoragedrivers backend, including collection layout, tenant scoping and error mapping. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinyagents-graph/src/checkpoint/README.md | 22 ++++++++++++++++++ crates/tinyagents-harness/src/store/README.md | 23 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/crates/tinyagents-graph/src/checkpoint/README.md b/crates/tinyagents-graph/src/checkpoint/README.md index c7f39a44d..8f28acd69 100644 --- a/crates/tinyagents-graph/src/checkpoint/README.md +++ b/crates/tinyagents-graph/src/checkpoint/README.md @@ -93,3 +93,25 @@ crashes mid-execution. - `DurabilityMode` trades persistence frequency for write volume — a mode that only checkpoints on interrupt/failure means a mid-run crash loses all progress since the last such boundary, not just the current node. + +## `DriverCheckpointer` (`storage-drivers` feature) + +`DriverCheckpointer` stores checkpoints, pending writes and execution leases in +a [tinystoragedrivers](https://github.com/tinyhumansai/tinystoragedrivers) +`DocumentStore`. Hosts use it to put graph durability on whichever backend they +opened, such as SQLite or MongoDB. The handle is bound to a tenant scope, so +tenants that share a thread id still never see each other's checkpoints. + +It uses four collections, all prefixed `graph` by default: + +- `_checkpoints`: one document per stored checkpoint, keyed by a + per-thread insertion sequence. Duplicate ids resolve to the latest write, + as in the append-only backends. +- `_threads`: the per-thread sequence counter, advanced with + compare-and-swap. +- `_writes`: merged pending writes per `(thread, namespace, checkpoint)`. +- `_leases`: per-thread execution leases, claimed, renewed and + released with compare-and-swap. + +It passes the same `testkit::conformance` contracts as the bundled backends: +checkpointer, writes, lineage and concurrency. diff --git a/crates/tinyagents-harness/src/store/README.md b/crates/tinyagents-harness/src/store/README.md index 93ff45322..831a6add6 100644 --- a/crates/tinyagents-harness/src/store/README.md +++ b/crates/tinyagents-harness/src/store/README.md @@ -75,3 +75,26 @@ The richer, TTL-aware, batch-oriented sibling trait — see its own README. writers. - None of the in-memory backends are durable: data is lost when the value is dropped. + +## Storage-driver backends (`storage-drivers` feature) + +`DriverStore` and `DriverAppendStore` adapt the harness `Store` and +`AppendStore` traits onto [tinystoragedrivers](https://github.com/tinyhumansai/tinystoragedrivers) +ports. A host opens one storage backend (SQLite on desktop, MongoDB in the cloud, +memory in tests), takes a scoped `DocumentStore` / `StreamStore` for the tenant, +and wraps it: + +```rust,ignore +let scoped = backend.for_scope(&Scope::new(agent_id)?)?; +let kv = DriverStore::new(scoped.documents().clone()); +let journal = DriverAppendStore::with_prefix(scoped.streams().clone(), "journal/"); +``` + +- `DriverStore` keeps every namespace in one collection (`harness_store` by + default) as `{ns, key, value}` documents, so namespaces and keys may hold any + characters, and `list` is an indexed query. +- `DriverAppendStore` maps each stream to a driver stream (optionally + prefixed). Driver offsets are already dense and zero-based, matching the + `AppendStore` contract. +- Driver `InvalidInput` errors surface as `TinyAgentsError::Validation`. Every + other driver error surfaces as `TinyAgentsError::Storage`. From c79c976548e1cab5b48267d09fd646883e634295 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:07:15 +0300 Subject: [PATCH 10/56] chore(vendor): add tinystoragedrivers submodule Register the tinystoragedrivers repository as a git submodule under vendor so its storage driver code can be consumed alongside the other vendored dependencies. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .gitmodules | 3 +++ vendor/tinystoragedrivers | 1 + 2 files changed, 4 insertions(+) create mode 160000 vendor/tinystoragedrivers diff --git a/.gitmodules b/.gitmodules index 3c1225634..c3c0ed1dc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "vendor/tinyliveagents"] path = vendor/tinyliveagents url = https://github.com/tinyhumansai/tinyliveagents.git +[submodule "vendor/tinystoragedrivers"] + path = vendor/tinystoragedrivers + url = https://github.com/tinyhumansai/tinystoragedrivers.git diff --git a/vendor/tinystoragedrivers b/vendor/tinystoragedrivers new file mode 160000 index 000000000..77d08547e --- /dev/null +++ b/vendor/tinystoragedrivers @@ -0,0 +1 @@ +Subproject commit 77d08547e01f45da8e0f746705ad26d218600c72 From 5857f2f2f1fa8fd6a927217b06cb254110eb0412 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:07:53 +0300 Subject: [PATCH 11/56] chore(deps): add tinystoragedrivers-core to the lockfile Record the new tinystoragedrivers-core 0.3.0 package and wire it into the dependency lists of the crates that now depend on it. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 6a635959e..1a7a1eed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1720,6 +1720,7 @@ dependencies = [ "tempfile", "tinyagents-harness", "tinyinference-llm", + "tinystoragedrivers-core", "tinytools", "tokio", "tracing", @@ -1755,6 +1756,7 @@ dependencies = [ "tinyinference-image", "tinyinference-llm", "tinyinference-video", + "tinystoragedrivers-core", "tinytools", "tinytools-agent", "tokio", @@ -1982,6 +1984,16 @@ dependencies = [ "url", ] +[[package]] +name = "tinystoragedrivers-core" +version = "0.3.0" +dependencies = [ + "async-trait", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "tinystr" version = "0.8.3" From d8f851180f549fc9461ca02a531b8a5ecd9d2a7a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:10:39 +0300 Subject: [PATCH 12/56] chore: apply rustfmt formatting across checkpoint and store drivers Reformatted the checkpoint and store driver modules and their tests to satisfy rustfmt, wrapping long expressions and reordering the DriverCheckpointer and SqliteCheckpointer re-exports. No behaviour changed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/checkpoint/drivers.rs | 35 ++++++++++------- .../src/checkpoint/drivers_tests.rs | 39 ++++++++++++++----- crates/tinyagents-graph/src/lib.rs | 4 +- .../tinyagents-harness/src/store/drivers.rs | 12 ++++-- .../src/store/drivers_tests.rs | 9 +++-- 5 files changed, 65 insertions(+), 34 deletions(-) diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs index 5348566b6..b5ba9e4f6 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -149,11 +149,7 @@ impl DriverCheckpointer { async fn next_seq(&self, thread: &str) -> Result { let id = key(&[thread]); for _ in 0..CAS_ATTEMPTS { - let current = self - .docs - .get(&self.threads, &id) - .await - .map_err(map_error)?; + let current = self.docs.get(&self.threads, &id).await.map_err(map_error)?; let (seq, precondition) = match ¤t { Some(found) => ( found.doc.get("next").and_then(Value::as_u64).unwrap_or(0), @@ -205,8 +201,10 @@ where { fn decode(stored: Versioned) -> Result> { let record = stored.doc.get("record").cloned().unwrap_or(Value::Null); - let mut checkpoint: Checkpoint = serde_json::from_value(record) - .map_err(|error| TinyAgentsError::Checkpoint(format!("storage driver: decode: {error}")))?; + let mut checkpoint: Checkpoint = + serde_json::from_value(record).map_err(|error| { + TinyAgentsError::Checkpoint(format!("storage driver: decode: {error}")) + })?; checkpoint.normalize(); Ok(checkpoint) } @@ -311,17 +309,19 @@ where return Ok(0); } self.declared().await?; - let filter = Filter::eq("thread", thread_id) - .and(Filter::one_of("checkpoint_id", ids.iter().map(String::as_str))); + let filter = Filter::eq("thread", thread_id).and(Filter::one_of( + "checkpoint_id", + ids.iter().map(String::as_str), + )); let removed = self .docs .delete_where(&self.checkpoints, &filter) .await .map_err(map_error)?; - self.drop_writes( - Filter::eq("thread", thread_id) - .and(Filter::one_of("checkpoint_id", ids.iter().map(String::as_str))), - ) + self.drop_writes(Filter::eq("thread", thread_id).and(Filter::one_of( + "checkpoint_id", + ids.iter().map(String::as_str), + ))) .await?; Ok(usize::try_from(removed).unwrap_or(usize::MAX)) } @@ -395,7 +395,8 @@ where found.unchanged() } }; - let expires_at_ms = now.saturating_add(u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX)); + let expires_at_ms = + now.saturating_add(u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX)); let doc = json!({ "owner": owner, "expires_at_ms": expires_at_ms }); match self.docs.put(&self.leases, &id, doc, precondition).await { Ok(_) => return Ok(true), @@ -424,7 +425,11 @@ where } let expires_at_ms = now.saturating_add(u64::try_from(ttl.as_millis()).unwrap_or(u64::MAX)); let doc = json!({ "owner": owner, "expires_at_ms": expires_at_ms }); - match self.docs.put(&self.leases, &id, doc, found.unchanged()).await { + match self + .docs + .put(&self.leases, &id, doc, found.unchanged()) + .await + { Ok(_) => Ok(true), Err(error) if error.kind() == ErrorKind::Conflict => Ok(false), Err(error) => Err(map_error(error)), diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs index 440a9c026..cdf1881c6 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -67,22 +67,43 @@ async fn leases_follow_the_claim_protocol() { let cp = checkpointer(); let minute = Duration::from_secs(60); assert!(cp.try_claim("t", "a", minute).await.unwrap()); - assert!(!cp.try_claim("t", "b", minute).await.unwrap(), "live lease is refused"); - assert!(cp.try_claim("t", "a", minute).await.unwrap(), "same owner re-claims"); + assert!( + !cp.try_claim("t", "b", minute).await.unwrap(), + "live lease is refused" + ); + assert!( + cp.try_claim("t", "a", minute).await.unwrap(), + "same owner re-claims" + ); assert!(cp.renew("t", "a", minute).await.unwrap()); - assert!(!cp.renew("t", "b", minute).await.unwrap(), "only the owner renews"); + assert!( + !cp.renew("t", "b", minute).await.unwrap(), + "only the owner renews" + ); assert!(!cp.renew("missing", "a", minute).await.unwrap()); cp.release("t", "b").await.unwrap(); - assert!(!cp.try_claim("t", "b", minute).await.unwrap(), "foreign release is a no-op"); + assert!( + !cp.try_claim("t", "b", minute).await.unwrap(), + "foreign release is a no-op" + ); cp.release("t", "a").await.unwrap(); cp.release("t", "a").await.unwrap(); - assert!(cp.try_claim("t", "b", minute).await.unwrap(), "released lease is free"); + assert!( + cp.try_claim("t", "b", minute).await.unwrap(), + "released lease is free" + ); assert!(cp.try_claim("z", "dead", Duration::ZERO).await.unwrap()); tokio::time::sleep(Duration::from_millis(5)).await; - assert!(!cp.renew("z", "dead", minute).await.unwrap(), "expired lease cannot renew"); - assert!(cp.try_claim("z", "new", minute).await.unwrap(), "expired lease is reclaimable"); + assert!( + !cp.renew("z", "dead", minute).await.unwrap(), + "expired lease cannot renew" + ); + assert!( + cp.try_claim("z", "new", minute).await.unwrap(), + "expired lease is reclaimable" + ); } #[test] @@ -105,9 +126,7 @@ async fn a_corrupt_record_is_a_checkpoint_error() { let storage = MemoryStorage::new(); let docs = docs(&storage, "local"); let cp = DriverCheckpointer::::new(Arc::clone(&docs)); - cp.put(sample("t", "c1", None, 1)) - .await - .unwrap(); + cp.put(sample("t", "c1", None, 1)).await.unwrap(); let stored = cp.thread_docs("t").await.unwrap().remove(0); docs.put( "graph_checkpoints", diff --git a/crates/tinyagents-graph/src/lib.rs b/crates/tinyagents-graph/src/lib.rs index e69439356..bf04be749 100644 --- a/crates/tinyagents-graph/src/lib.rs +++ b/crates/tinyagents-graph/src/lib.rs @@ -58,10 +58,10 @@ pub use channel::{ Barrier, BinaryAggregate, Channel, ChannelSet, ChannelState, ChannelUpdate, ChannelWrite, Delta, Ephemeral, LastValue, Messages, NamedBarrier, ReducerRegistry, Topic, Untracked, }; -#[cfg(feature = "sqlite")] -pub use checkpoint::SqliteCheckpointer; #[cfg(feature = "storage-drivers")] pub use checkpoint::DriverCheckpointer; +#[cfg(feature = "sqlite")] +pub use checkpoint::SqliteCheckpointer; pub use checkpoint::{ BarrierArrivals, CHECKPOINT_FORMAT_VERSION, Checkpoint, CheckpointConfig, CheckpointMetadata, CheckpointSource, CheckpointTuple, Checkpointer, CompletedTask, DurabilityMode, diff --git a/crates/tinyagents-harness/src/store/drivers.rs b/crates/tinyagents-harness/src/store/drivers.rs index 1bae8bc12..f5b922d3f 100644 --- a/crates/tinyagents-harness/src/store/drivers.rs +++ b/crates/tinyagents-harness/src/store/drivers.rs @@ -77,8 +77,8 @@ impl DriverStore { async fn declared(&self) -> Result<()> { self.declared .get_or_try_init(|| async { - let spec = CollectionSpec::new(&self.collection) - .index(IndexSpec::new("by_ns", ["ns"])); + let spec = + CollectionSpec::new(&self.collection).index(IndexSpec::new("by_ns", ["ns"])); self.docs.ensure_collection(&spec).await.map_err(map_error) }) .await @@ -136,7 +136,13 @@ impl Store for DriverStore { .map_err(map_error)?; Ok(found .into_iter() - .filter_map(|stored| stored.doc.get("key").and_then(Value::as_str).map(str::to_owned)) + .filter_map(|stored| { + stored + .doc + .get("key") + .and_then(Value::as_str) + .map(str::to_owned) + }) .collect()) } } diff --git a/crates/tinyagents-harness/src/store/drivers_tests.rs b/crates/tinyagents-harness/src/store/drivers_tests.rs index a2d7f0635..1ca786b6e 100644 --- a/crates/tinyagents-harness/src/store/drivers_tests.rs +++ b/crates/tinyagents-harness/src/store/drivers_tests.rs @@ -3,9 +3,7 @@ use crate::store::conformance::run_store_conformance; use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; fn scoped(storage: &MemoryStorage, scope: &str) -> tinystoragedrivers_core::ScopedStorage { - storage - .for_scope(&Scope::new(scope).unwrap()) - .unwrap() + storage.for_scope(&Scope::new(scope).unwrap()).unwrap() } #[tokio::test] @@ -48,7 +46,10 @@ async fn a_list_follows_every_page() { let storage = MemoryStorage::new(); let store = DriverStore::new(Arc::clone(scoped(&storage, "local").documents())); for i in 0..1_005 { - store.put("big", &format!("k{i:04}"), json!(i)).await.unwrap(); + store + .put("big", &format!("k{i:04}"), json!(i)) + .await + .unwrap(); } assert_eq!(store.list("big").await.unwrap().len(), 1_005); } From 5b66daa06ef3009d2e1541a7e4ddc6c844601182 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:19:06 +0300 Subject: [PATCH 13/56] feat(session): add in-memory session store Add an in-memory implementation of the session store port so sessions can be persisted without an external backend, which is useful for tests and ephemeral runs. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/Cargo.toml | 9 +++++++++ crates/tinyagents-session/src/port/memory/mod.rs | 10 +++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/tinyagents-session/Cargo.toml b/crates/tinyagents-session/Cargo.toml index de3587e74..180c5ec01 100644 --- a/crates/tinyagents-session/Cargo.toml +++ b/crates/tinyagents-session/Cargo.toml @@ -28,6 +28,8 @@ tracing = { workspace = true } tokio = { workspace = true, default-features = true, features = ["sync"] } # `threads` names its atomic-rewrite temp files `.conversations-.tmp`. uuid = { workspace = true } +# Session stores over `tinystoragedrivers` ports (`storage-drivers` feature). +tinystoragedrivers-core = { workspace = true, optional = true, features = ["blocking"] } [features] default = [] @@ -35,6 +37,13 @@ default = [] # dependency above). This feature is retained as a no-op so downstream # feature forwards keep compiling. tracing = ["tinyagents-harness/tracing"] +# A `SessionStoreProvider` over `tinystoragedrivers` ports, so a host can keep +# every agent's transcripts, turn states, records and journal in whichever +# backend it opened (SQLite, MongoDB, memory, ...). +storage-drivers = [ + "dep:tinystoragedrivers-core", + "tinyagents-harness/storage-drivers", +] [dev-dependencies] tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/tinyagents-session/src/port/memory/mod.rs b/crates/tinyagents-session/src/port/memory/mod.rs index 6a7d1779a..3bb9ec0bd 100644 --- a/crates/tinyagents-session/src/port/memory/mod.rs +++ b/crates/tinyagents-session/src/port/memory/mod.rs @@ -21,10 +21,10 @@ use crate::transcript::{ }; use crate::turn_state::{TurnLifecycle, TurnState}; -const MAX_GENERATIONS: u32 = 4096; +pub(super) const MAX_GENERATIONS: u32 = 4096; /// Completed turns kept per thread, as the on-disk store keeps them. -const COMPLETED_RETENTION: usize = 20; +pub(super) const COMPLETED_RETENTION: usize = 20; /// A [`SessionStoreProvider`] keeping every agent's stores in memory, each /// agent's apart from every other's. @@ -357,14 +357,14 @@ impl InMemoryTurnStates { } /// Newest first: by `started_at`, then `updated_at`. -fn newest_first(a: &TurnState, b: &TurnState) -> Ordering { +pub(super) fn newest_first(a: &TurnState, b: &TurnState) -> Ordering { compare_rfc3339(&b.started_at, &a.started_at) .then_with(|| compare_rfc3339(&b.updated_at, &a.updated_at)) } /// Completed-turn retention follows completion time, as the durable store /// does. Listing the latest live turn still follows its start time. -fn completed_newest_first(a: &TurnState, b: &TurnState) -> Ordering { +pub(super) fn completed_newest_first(a: &TurnState, b: &TurnState) -> Ordering { compare_rfc3339(&b.updated_at, &a.updated_at) .then_with(|| compare_rfc3339(&b.started_at, &a.started_at)) } @@ -383,7 +383,7 @@ fn key(thread_id: &str, request_id: &str) -> (String, String) { (thread_id.to_string(), request_id.to_string()) } -fn is_terminal(state: &TurnState) -> bool { +pub(super) fn is_terminal(state: &TurnState) -> bool { matches!( state.lifecycle, TurnLifecycle::Interrupted | TurnLifecycle::Completed From 0fe4ecee97c47021135190fed5289124c0f6ca1b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:19:36 +0300 Subject: [PATCH 14/56] chore(session): remove unused drivers module The drivers module in the session port no longer has any consumers, so it has been dropped to keep the port surface minimal. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/mod.rs | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 crates/tinyagents-session/src/port/drivers/mod.rs diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs new file mode 100644 index 000000000..c7ff119e7 --- /dev/null +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -0,0 +1,242 @@ +//! A [`SessionStoreProvider`] over `tinystoragedrivers` ports. +//! +//! A host opens one storage backend (SQLite on a desktop, MongoDB in the +//! cloud, memory in tests) and hands it to [`DriverSessionStores`]. Each +//! agent's stores then live in that backend under the agent's own +//! [`Scope`], so one shared database serves every agent and the driver, not +//! this crate, keeps them apart: a handle bound to one scope cannot name +//! another scope's records. +//! +//! # What goes where +//! +//! | Store | Backend shape | +//! | --- | --- | +//! | transcripts | `session_transcripts` (one index document per stem) and `session_transcript_entries` (an append-only log per stem) | +//! | turn states | `session_turn_states`, one document per turn | +//! | key-value | `session_kv`, through the harness [`DriverStore`] | +//! | journal | streams prefixed `session_journal/`, through the harness [`DriverAppendStore`] | +//! +//! # Sync seams +//! +//! The transcript and turn-state traits are synchronous, so their +//! implementations here run each driver call on a [`Blocking`] bridge: a +//! dedicated runtime thread, which works from any caller (inside a tokio +//! runtime or not) and never needs `block_in_place`. +//! +//! # Scopes +//! +//! An agent id that is a valid [`Scope`] is used as is. Any other id (empty, +//! over-long, or holding whitespace) maps to `sha256:` of the id, so +//! every agent still gets a scope of its own and the mapping never changes. + +mod refused; +mod transcripts; +mod turn_states; + +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, PoisonError}; + +use sha2::{Digest, Sha256}; +use tinyagents_harness::store::{DriverAppendStore, DriverStore}; +use tinystoragedrivers_core::{Blocking, Scope, ScopedStorage, StorageBackend, StorageError}; + +use super::{AgentStores, SessionStoreProvider}; + +pub use transcripts::{DriverTranscriptHistory, DriverTranscriptLocator}; +pub use turn_states::DriverTurnStates; + +/// Collection holding each agent's key-value records. +const KV_COLLECTION: &str = "session_kv"; +/// Prefix of each agent's journal streams. +const JOURNAL_PREFIX: &str = "session_journal/"; + +/// A [`SessionStoreProvider`] keeping every agent's stores in one +/// `tinystoragedrivers` backend, each agent in its own [`Scope`]. +pub struct DriverSessionStores { + backend: Arc, + bridge: Blocking, + agents: Mutex>, + recover_on_open: bool, + recovered: Mutex>, +} + +impl std::fmt::Debug for DriverSessionStores { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let agents = self.agents.lock().unwrap_or_else(PoisonError::into_inner); + f.debug_struct("DriverSessionStores") + .field("driver", &self.backend.driver()) + .field("agents", &agents.len()) + .field("recover_on_open", &self.recover_on_open) + .finish_non_exhaustive() + } +} + +impl DriverSessionStores { + /// A provider over `backend`, with its own [`Blocking`] bridge. + /// + /// # Errors + /// + /// When the bridge's runtime thread cannot start. + pub fn new(backend: Arc) -> Result { + Ok(Self::with_bridge(backend, Blocking::new()?)) + } + + /// A provider over `backend` that runs its synchronous seams on `bridge`, + /// so several providers (or other sync adapters) can share one thread. + pub fn with_bridge(backend: Arc, bridge: Blocking) -> Self { + Self { + backend, + bridge, + agents: Mutex::new(HashMap::new()), + recover_on_open: false, + recovered: Mutex::new(HashSet::new()), + } + } + + /// Marks an agent's in-flight turns interrupted the first time this + /// provider opens its stores. + /// + /// For a single-process host (the desktop app): there, a turn still in + /// flight when the process starts can only be one an earlier process + /// left behind. A host that runs several processes against one database + /// must leave this off, since another process may own those turns. + #[must_use] + pub fn recover_on_open(mut self, enabled: bool) -> Self { + self.recover_on_open = enabled; + self + } + + /// The scope agent `agent_id` is stored under. + pub fn scope_for(agent_id: &str) -> Scope { + Scope::new(agent_id).unwrap_or_else(|_| { + let digest = Sha256::digest(agent_id.as_bytes()); + Scope::new(format!("sha256:{}", hex::encode(digest))) + .expect("a sha256 hex scope is always valid") + }) + } + + /// The stores of `agent_id`, or the error that kept the backend from + /// binding its scope. + /// + /// # Errors + /// + /// Whatever [`StorageBackend::for_scope`] reports. + pub fn try_for_agent(&self, agent_id: &str) -> Result { + if let Some(stores) = self + .agents + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(agent_id) + { + return Ok(stores.clone()); + } + let scoped = self.backend.for_scope(&Self::scope_for(agent_id))?; + let stores = self.build(&scoped); + if self.recover_on_open + && self + .recovered + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(agent_id.to_string()) + { + let now = chrono::Utc::now().to_rfc3339(); + match stores.turn_states.mark_all_interrupted(&now) { + Ok(count) if count > 0 => tracing::info!( + target: "tinyagents_session::port::drivers", + agent_id, + count, + "[session-store] marked turns left in flight interrupted" + ), + Ok(_) => {} + Err(error) => tracing::warn!( + target: "tinyagents_session::port::drivers", + agent_id, + %error, + "[session-store] could not recover in-flight turns" + ), + } + } + Ok(self + .agents + .lock() + .unwrap_or_else(PoisonError::into_inner) + .entry(agent_id.to_string()) + .or_insert(stores) + .clone()) + } + + fn build(&self, scoped: &ScopedStorage) -> AgentStores { + let docs = Arc::clone(scoped.documents()); + let label = format!("{}://{}", scoped.driver(), scoped.scope()); + AgentStores { + transcripts: Arc::new(DriverTranscriptLocator::new( + Arc::clone(&docs), + self.bridge.clone(), + label, + )), + turn_states: Arc::new(DriverTurnStates::new( + Arc::clone(&docs), + self.bridge.clone(), + )), + kv: Arc::new(DriverStore::with_collection(docs, KV_COLLECTION)), + journal: Arc::new(DriverAppendStore::with_prefix( + Arc::clone(scoped.streams()), + JOURNAL_PREFIX, + )), + } + } +} + +impl SessionStoreProvider for DriverSessionStores { + /// The stores of `agent_id`. + /// + /// Fails closed: when the backend cannot bind the agent's scope, the + /// stores returned refuse every call with that error rather than fall + /// back to somewhere the agent's data does not belong. They are not + /// cached, so the next call tries the backend again. + fn for_agent(&self, agent_id: &str) -> AgentStores { + self.try_for_agent(agent_id).unwrap_or_else(|error| { + tracing::error!( + target: "tinyagents_session::port::drivers", + agent_id, + %error, + "[session-store] backend refused the agent's scope" + ); + refused::stores(&error) + }) + } + + /// Marks in-flight turns interrupted for every agent this provider has + /// opened. A backend cannot enumerate its scopes, so agents not yet + /// opened are recovered by [`Self::recover_on_open`] instead. + fn recover(&self) -> anyhow::Result<()> { + let now = chrono::Utc::now().to_rfc3339(); + let agents: Vec = self + .agents + .lock() + .unwrap_or_else(PoisonError::into_inner) + .values() + .cloned() + .collect(); + for stores in agents { + stores + .turn_states + .mark_all_interrupted(&now) + .map_err(anyhow::Error::msg)?; + } + Ok(()) + } + + fn destination_key(&self) -> Option { + Some(format!("{}://{:p}", self.backend.driver(), Arc::as_ptr(&self.backend))) + } +} + +/// Map a driver failure onto the `String` errors of the turn-state seam. +fn describe(error: &StorageError) -> String { + error.to_string() +} + +#[cfg(test)] +#[path = "mod_tests.rs"] +mod tests; From 0916af8e70e9abd832ed2f4eeb56299c9e58c283 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:19:58 +0300 Subject: [PATCH 15/56] fix(session): reject refused driver calls with an error The refused driver now returns an error instead of silently succeeding when its methods are invoked, so callers that reach it during a session fail loudly rather than continuing with no effect. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/refused.rs | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 crates/tinyagents-session/src/port/drivers/refused.rs diff --git a/crates/tinyagents-session/src/port/drivers/refused.rs b/crates/tinyagents-session/src/port/drivers/refused.rs new file mode 100644 index 000000000..92eb89280 --- /dev/null +++ b/crates/tinyagents-session/src/port/drivers/refused.rs @@ -0,0 +1,130 @@ +//! Stores that refuse every call: what an agent gets when the backend could +//! not bind its scope, so the failure surfaces on use instead of the agent's +//! data landing somewhere it does not belong. + +use std::sync::Arc; + +use serde_json::Value; +use tinystoragedrivers_core::{ + Capabilities, CollectionSpec, DocumentStore, ErrorKind, Filter, Page, Precondition, Query, + Result, Sort, StorageError, StreamEntry, StreamStore, Version, Versioned, async_trait, +}; + +use super::super::AgentStores; +use super::{DriverAppendStore, DriverStore, DriverTranscriptLocator, DriverTurnStates}; + +/// A document and stream store whose every call fails with one error. +#[derive(Debug)] +pub(super) struct Refused { + kind: ErrorKind, + message: String, +} + +impl Refused { + fn error(&self) -> StorageError { + StorageError::new(self.kind, self.message.clone()) + } +} + +/// Stores over [`Refused`], reporting `error` on every call. +pub(super) fn stores(error: &StorageError) -> AgentStores { + let refused = Arc::new(Refused { + kind: error.kind(), + message: format!("session stores unavailable: {}", error.message()), + }); + let bridge = tinystoragedrivers_core::Blocking::new(); + let docs: Arc = refused.clone(); + let streams: Arc = refused; + AgentStores { + transcripts: Arc::new(DriverTranscriptLocator::refused(Arc::clone(&docs), bridge.as_ref().ok().cloned())), + turn_states: Arc::new(DriverTurnStates::refused(Arc::clone(&docs), bridge.ok())), + kv: Arc::new(DriverStore::new(docs)), + journal: Arc::new(DriverAppendStore::new(streams)), + } +} + +#[async_trait] +impl DocumentStore for Refused { + fn capabilities(&self) -> Capabilities { + Capabilities::default() + } + + async fn ensure_collection(&self, _spec: &CollectionSpec) -> Result<()> { + Err(self.error()) + } + + async fn get(&self, _collection: &str, _id: &str) -> Result>> { + Err(self.error()) + } + + async fn put( + &self, + _collection: &str, + _id: &str, + _doc: Value, + _precondition: Precondition, + ) -> Result { + Err(self.error()) + } + + async fn delete(&self, _collection: &str, _id: &str, _pre: Precondition) -> Result { + Err(self.error()) + } + + async fn query(&self, _collection: &str, _query: &Query) -> Result>> { + Err(self.error()) + } + + async fn count(&self, _collection: &str, _filter: &Filter) -> Result { + Err(self.error()) + } + + async fn delete_where(&self, _collection: &str, _filter: &Filter) -> Result { + Err(self.error()) + } + + async fn claim( + &self, + _collection: &str, + _filter: &Filter, + _sort: &[Sort], + _patch: &Value, + ) -> Result>> { + Err(self.error()) + } + + async fn drop_collection(&self, _collection: &str) -> Result<()> { + Err(self.error()) + } +} + +#[async_trait] +impl StreamStore for Refused { + async fn append(&self, _stream: &str, _value: Value) -> Result { + Err(self.error()) + } + + async fn append_batch(&self, _stream: &str, _values: Vec) -> Result { + Err(self.error()) + } + + async fn read_window(&self, _stream: &str, _from: u64, _limit: usize) -> Result> { + Err(self.error()) + } + + async fn len(&self, _stream: &str) -> Result { + Err(self.error()) + } + + async fn truncate_before(&self, _stream: &str, _offset: u64) -> Result { + Err(self.error()) + } + + async fn delete_stream(&self, _stream: &str) -> Result { + Err(self.error()) + } + + async fn streams(&self, _prefix: &str) -> Result> { + Err(self.error()) + } +} From 539d5f01bf7e4a31bfbeb4f3f675df286077ba0d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:20:07 +0300 Subject: [PATCH 16/56] refactor(session): extract refused driver into its own module Move the refused driver implementation out of the drivers module into a dedicated refused submodule. This keeps the module tree aligned with the other driver implementations and makes the port layout easier to navigate. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/port/drivers/mod.rs | 7 +------ .../tinyagents-session/src/port/drivers/refused.rs | 13 ++++++++----- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index c7ff119e7..4f2425402 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -202,7 +202,7 @@ impl SessionStoreProvider for DriverSessionStores { %error, "[session-store] backend refused the agent's scope" ); - refused::stores(&error) + refused::stores(&error, &self.bridge) }) } @@ -232,11 +232,6 @@ impl SessionStoreProvider for DriverSessionStores { } } -/// Map a driver failure onto the `String` errors of the turn-state seam. -fn describe(error: &StorageError) -> String { - error.to_string() -} - #[cfg(test)] #[path = "mod_tests.rs"] mod tests; diff --git a/crates/tinyagents-session/src/port/drivers/refused.rs b/crates/tinyagents-session/src/port/drivers/refused.rs index 92eb89280..3d96b3ad6 100644 --- a/crates/tinyagents-session/src/port/drivers/refused.rs +++ b/crates/tinyagents-session/src/port/drivers/refused.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use serde_json::Value; use tinystoragedrivers_core::{ - Capabilities, CollectionSpec, DocumentStore, ErrorKind, Filter, Page, Precondition, Query, + Blocking, Capabilities, CollectionSpec, DocumentStore, ErrorKind, Filter, Page, Precondition, Query, Result, Sort, StorageError, StreamEntry, StreamStore, Version, Versioned, async_trait, }; @@ -27,17 +27,20 @@ impl Refused { } /// Stores over [`Refused`], reporting `error` on every call. -pub(super) fn stores(error: &StorageError) -> AgentStores { +pub(super) fn stores(error: &StorageError, bridge: &Blocking) -> AgentStores { let refused = Arc::new(Refused { kind: error.kind(), message: format!("session stores unavailable: {}", error.message()), }); - let bridge = tinystoragedrivers_core::Blocking::new(); let docs: Arc = refused.clone(); let streams: Arc = refused; AgentStores { - transcripts: Arc::new(DriverTranscriptLocator::refused(Arc::clone(&docs), bridge.as_ref().ok().cloned())), - turn_states: Arc::new(DriverTurnStates::refused(Arc::clone(&docs), bridge.ok())), + transcripts: Arc::new(DriverTranscriptLocator::new( + Arc::clone(&docs), + bridge.clone(), + "refused", + )), + turn_states: Arc::new(DriverTurnStates::new(Arc::clone(&docs), bridge.clone())), kv: Arc::new(DriverStore::new(docs)), journal: Arc::new(DriverAppendStore::new(streams)), } From 855e9bcdeecd588d9283040edeb4ee4eb73d46fe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:20:49 +0300 Subject: [PATCH 17/56] refactor(session): restructure turn state driver Reorganize the turn state driver to clarify the state transitions and separate the handling of each turn phase. No behaviour changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/turn_states.rs | 400 ++++++++++++++++++ 1 file changed, 400 insertions(+) create mode 100644 crates/tinyagents-session/src/port/drivers/turn_states.rs diff --git a/crates/tinyagents-session/src/port/drivers/turn_states.rs b/crates/tinyagents-session/src/port/drivers/turn_states.rs new file mode 100644 index 000000000..483cd3f17 --- /dev/null +++ b/crates/tinyagents-session/src/port/drivers/turn_states.rs @@ -0,0 +1,400 @@ +//! [`TurnStates`] over a driver [`DocumentStore`]. +//! +//! One document per turn in [`COLLECTION`], keyed by `(thread, request)`: +//! `{ "thread_id", "request_id", "lifecycle", "turn": }`, with +//! the thread indexed so a thread's turns are one query. Every conditional +//! change (a write that must not clobber a completed turn, a settle, an +//! interruption sweep) is a compare-and-swap on the document's version, so +//! two processes sharing a database cannot lose each other's updates. +//! +//! Semantics match [`InMemoryTurnStates`](crate::port::InMemoryTurnStates): +//! "latest" is the newest `started_at`, completed turns are kept up to +//! [`COMPLETED_RETENTION`] per thread, and a completed turn is never +//! overwritten by a conditional write. + +use std::cmp::Ordering; +use std::collections::HashMap; +use std::future::Future; +use std::sync::Arc; + +use serde_json::{Value, json}; +use tinystoragedrivers_core::{ + Blocking, CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, Filter, IndexSpec, + Precondition, Query, StorageError, Versioned, +}; +use tokio::sync::OnceCell; + +use super::super::TurnStates; +use super::super::memory::{COMPLETED_RETENTION, completed_newest_first, is_terminal, newest_first}; +use super::transcripts::doc_key; +use crate::turn_state::{TurnLifecycle, TurnState}; + +/// The collection turn snapshots live in. +pub(super) const COLLECTION: &str = "session_turn_states"; + +/// Compare-and-swap attempts before a contended update gives up. +const CAS_ATTEMPTS: usize = 64; + +/// [`TurnStates`] stored in a driver [`DocumentStore`]. +#[derive(Debug, Clone)] +pub struct DriverTurnStates { + inner: Arc, + bridge: Blocking, +} + +#[derive(Debug)] +struct Inner { + docs: Arc, + declared: OnceCell<()>, +} + +impl DriverTurnStates { + /// Turn states in `docs`, with synchronous calls run on `bridge`. + pub fn new(docs: Arc, bridge: Blocking) -> Self { + Self { + inner: Arc::new(Inner { + docs, + declared: OnceCell::new(), + }), + bridge, + } + } + + /// Runs `op` against the store on the bridge. + fn run(&self, op: F) -> Result + where + F: FnOnce(Arc) -> Fut, + Fut: Future> + Send + 'static, + T: Send + 'static, + { + let inner = Arc::clone(&self.inner); + let future = op(inner); + match self.bridge.run(future) { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) | Err(error) => Err(error.to_string()), + } + } +} + +fn id(thread_id: &str, request_id: &str) -> String { + doc_key(&[thread_id, request_id]) +} + +fn encode(state: &TurnState) -> Result { + let turn = serde_json::to_value(state) + .map_err(|error| StorageError::serialization(error.to_string()))?; + let lifecycle = serde_json::to_value(state.lifecycle) + .map_err(|error| StorageError::serialization(error.to_string()))?; + Ok(json!({ + "thread_id": state.thread_id, + "request_id": state.request_id, + "lifecycle": lifecycle, + "turn": turn, + })) +} + +fn decode(stored: Versioned) -> Result, StorageError> { + let Versioned { id, version, doc } = stored; + let turn = doc + .get("turn") + .cloned() + .ok_or_else(|| StorageError::serialization(format!("turn state {id} has no body")))?; + let turn = serde_json::from_value(turn).map_err(|error| { + StorageError::serialization(format!("turn state {id} is unreadable: {error}")) + })?; + Ok(Versioned { + id, + version, + doc: turn, + }) +} + +impl Inner { + async fn declared(&self) -> Result<(), StorageError> { + self.declared + .get_or_try_init(|| async { + let spec = CollectionSpec::new(COLLECTION) + .index(IndexSpec::new("by_thread", ["thread_id"])); + self.docs.ensure_collection(&spec).await + }) + .await + .map(|_| ()) + } + + async fn get_turn( + &self, + thread_id: &str, + request_id: &str, + ) -> Result>, StorageError> { + self.declared().await?; + self.docs + .get(COLLECTION, &id(thread_id, request_id)) + .await? + .map(decode) + .transpose() + } + + async fn query(&self, filter: Filter) -> Result>, StorageError> { + self.declared().await?; + self.docs + .query_all(COLLECTION, &Query::filter(filter).limit(1_000)) + .await? + .into_iter() + .map(decode) + .collect() + } + + async fn thread(&self, thread_id: &str) -> Result, StorageError> { + let mut turns: Vec = self + .query(Filter::eq("thread_id", thread_id)) + .await? + .into_iter() + .map(|stored| stored.doc) + .collect(); + turns.sort_by(newest_first); + Ok(turns) + } + + async fn write(&self, state: &TurnState, pre: Precondition) -> Result<(), StorageError> { + self.declared().await?; + self.docs + .put( + COLLECTION, + &id(&state.thread_id, &state.request_id), + encode(state)?, + pre, + ) + .await + .map(|_| ()) + } + + /// Drops `thread_id`'s completed turns beyond the newest + /// [`COMPLETED_RETENTION`]. + async fn prune_completed(&self, thread_id: &str) -> Result<(), StorageError> { + let mut completed: Vec = self + .thread(thread_id) + .await? + .into_iter() + .filter(|turn| turn.lifecycle == TurnLifecycle::Completed) + .collect(); + completed.sort_by(completed_newest_first); + for stale in completed.iter().skip(COMPLETED_RETENTION) { + self.docs + .delete( + COLLECTION, + &id(&stale.thread_id, &stale.request_id), + Precondition::None, + ) + .await?; + } + Ok(()) + } + + /// Applies `change` to one turn under compare-and-swap; returns whether + /// it changed anything. `change` answers `None` to leave the turn alone. + async fn update( + &self, + thread_id: &str, + request_id: &str, + change: impl Fn(&TurnState) -> Option, + ) -> Result, StorageError> { + for _ in 0..CAS_ATTEMPTS { + let Some(stored) = self.get_turn(thread_id, request_id).await? else { + return Ok(None); + }; + let Some(next) = change(&stored.doc) else { + return Ok(None); + }; + match self.write(&next, stored.unchanged()).await { + Ok(()) => return Ok(Some(next)), + Err(error) if is_race(&error) => {} + Err(error) => return Err(error), + } + } + Err(contended(thread_id, request_id)) + } +} + +/// Whether `error` means another writer got there first. +fn is_race(error: &StorageError) -> bool { + matches!(error.kind(), ErrorKind::Conflict | ErrorKind::NotFound) +} + +fn contended(thread_id: &str, request_id: &str) -> StorageError { + StorageError::conflict(format!( + "turn {thread_id}/{request_id} kept changing under {CAS_ATTEMPTS} attempts" + )) +} + +/// `turn` marked interrupted as of `now`. +fn interrupted(turn: &TurnState, now: &str) -> Option { + if is_terminal(turn) { + return None; + } + let mut next = turn.clone(); + next.lifecycle = TurnLifecycle::Interrupted; + next.updated_at = now.to_string(); + next.active_tool = None; + next.active_subagent = None; + Some(next) +} + +impl TurnStates for DriverTurnStates { + fn put(&self, state: &TurnState) -> Result<(), String> { + let state = state.clone(); + self.run(|inner| async move { + inner.write(&state, Precondition::None).await?; + if state.lifecycle == TurnLifecycle::Completed { + inner.prune_completed(&state.thread_id).await?; + } + Ok(()) + }) + } + + fn put_unless_completed(&self, state: &TurnState) -> Result { + let state = state.clone(); + self.run(|inner| async move { + for _ in 0..CAS_ATTEMPTS { + let pre = match inner.get_turn(&state.thread_id, &state.request_id).await? { + Some(stored) if stored.doc.lifecycle == TurnLifecycle::Completed => { + return Ok(false); + } + Some(stored) => stored.unchanged(), + None => Precondition::Absent, + }; + match inner.write(&state, pre).await { + Ok(()) => { + if state.lifecycle == TurnLifecycle::Completed { + inner.prune_completed(&state.thread_id).await?; + } + return Ok(true); + } + Err(error) if is_race(&error) || error.kind() == ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + Err(contended(&state.thread_id, &state.request_id)) + }) + } + + fn get(&self, thread_id: &str) -> Result, String> { + Ok(self.list_thread(thread_id)?.into_iter().next()) + } + + fn get_turn(&self, thread_id: &str, request_id: &str) -> Result, String> { + let (thread_id, request_id) = (thread_id.to_string(), request_id.to_string()); + self.run(|inner| async move { + Ok(inner + .get_turn(&thread_id, &request_id) + .await? + .map(|stored| stored.doc)) + }) + } + + fn delete(&self, thread_id: &str) -> Result { + let thread_id = thread_id.to_string(); + self.run(|inner| async move { + inner.declared().await?; + let removed = inner + .docs + .delete_where(COLLECTION, &Filter::eq("thread_id", thread_id)) + .await?; + Ok(removed > 0) + }) + } + + fn delete_turn(&self, thread_id: &str, request_id: &str) -> Result { + let key = id(thread_id, request_id); + self.run(|inner| async move { + inner.declared().await?; + inner.docs.delete(COLLECTION, &key, Precondition::None).await + }) + } + + fn list(&self) -> Result, String> { + self.run(|inner| async move { + let mut latest: HashMap = HashMap::new(); + for turn in inner.query(Filter::All).await? { + let turn = turn.doc; + match latest.get(&turn.thread_id) { + Some(kept) if newest_first(&turn, kept) != Ordering::Less => {} + _ => { + latest.insert(turn.thread_id.clone(), turn); + } + } + } + Ok(latest.into_values().collect()) + }) + } + + fn list_thread(&self, thread_id: &str) -> Result, String> { + let thread_id = thread_id.to_string(); + self.run(|inner| async move { inner.thread(&thread_id).await }) + } + + fn clear_all(&self) -> Result { + self.run(|inner| async move { + inner.declared().await?; + let removed = inner.docs.delete_where(COLLECTION, &Filter::All).await?; + Ok(usize::try_from(removed).unwrap_or(usize::MAX)) + }) + } + + fn mark_all_interrupted(&self, now_rfc3339: &str) -> Result { + let now = now_rfc3339.to_string(); + self.run(|inner| async move { + let mut count = 0; + for stored in inner.query(Filter::All).await? { + let turn = stored.doc; + if is_terminal(&turn) { + continue; + } + if inner + .update(&turn.thread_id, &turn.request_id, |current| { + interrupted(current, &now) + }) + .await? + .is_some() + { + count += 1; + } + } + Ok(count) + }) + } + + fn settle_turn( + &self, + thread_id: &str, + request_id: &str, + lifecycle: TurnLifecycle, + now_rfc3339: &str, + ) -> Result { + let (thread_id, request_id) = (thread_id.to_string(), request_id.to_string()); + let now = now_rfc3339.to_string(); + self.run(|inner| async move { + let settled = inner + .update(&thread_id, &request_id, |current| { + if is_terminal(current) { + return None; + } + let mut next = current.clone(); + next.lifecycle = lifecycle; + next.phase = None; + next.active_tool = None; + next.active_subagent = None; + next.updated_at.clone_from(&now); + Some(next) + }) + .await?; + if settled.is_some() && lifecycle == TurnLifecycle::Completed { + inner.prune_completed(&thread_id).await?; + } + Ok(settled.is_some()) + }) + } +} + +#[cfg(test)] +#[path = "turn_states_tests.rs"] +mod tests; From 4ae572b20c4782b865aea826afcf01962e2debaa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:21:45 +0300 Subject: [PATCH 18/56] refactor(session): restructure turn state driver Reorganize the turn state driver to clarify the state transitions and separate the handling of each turn phase. No behaviour changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/port/drivers/turn_states.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/turn_states.rs b/crates/tinyagents-session/src/port/drivers/turn_states.rs index 483cd3f17..3a14b59e5 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states.rs @@ -217,7 +217,7 @@ impl Inner { /// Whether `error` means another writer got there first. fn is_race(error: &StorageError) -> bool { - matches!(error.kind(), ErrorKind::Conflict | ErrorKind::NotFound) + error.kind() == ErrorKind::Conflict } fn contended(thread_id: &str, request_id: &str) -> StorageError { @@ -269,7 +269,7 @@ impl TurnStates for DriverTurnStates { } return Ok(true); } - Err(error) if is_race(&error) || error.kind() == ErrorKind::AlreadyExists => {} + Err(error) if is_race(&error) => {} Err(error) => return Err(error), } } From 481137eb15e80bc431f95820a82a429848c439f4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:23:13 +0300 Subject: [PATCH 19/56] refactor(tinyagents-session): restructure transcript driver port Reorganized the transcript driver port to clarify the boundary between the port interface and its implementations. No behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts.rs | 890 ++++++++++++++++++ 1 file changed, 890 insertions(+) create mode 100644 crates/tinyagents-session/src/port/drivers/transcripts.rs diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs new file mode 100644 index 000000000..900b6c400 --- /dev/null +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -0,0 +1,890 @@ +//! [`TranscriptLocator`] and [`TranscriptHistory`] over a driver +//! [`DocumentStore`]. +//! +//! # Layout +//! +//! Each transcript stem is an append-only log of entries in +//! [`ENTRIES`], one document per write, numbered `0, 1, 2, …`: +//! +//! ```text +//! { "stem", "seq", "written", "meta"?, "set"?, "extend"?, "tools"?, +//! "partial"?, "request_id"?, "clear"?, "seal"? } +//! ``` +//! +//! Replaying the entries in order rebuilds the transcript: `set` replaces the +//! logical messages (a first write or a compaction), `extend` appends to them +//! (an ordinary turn only stores its new rows), `partial` records a +//! display-only partial, `clear` empties both, and `seal` closes the +//! generation. A writer claims the next number with an insert-only write, so +//! two writers racing for the same transcript (two processes sharing a +//! database) are serialized: the loser re-reads, sees the winner's entry, and +//! decides again — which is also how a write that races a seal is refused. +//! +//! [`INDEX`] holds one small document per stem with the lookup fields +//! (`thread_id`, `agent_id`, `agent_name`, `subagent`, `created_at`), so +//! "newest root transcript of this thread" is one indexed query instead of a +//! scan of every log. It is refreshed after each write that changes them. A +//! successor generation is reserved there before its predecessor is sealed. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use tinystoragedrivers_core::{ + Blocking, CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, Filter, IndexSpec, + Precondition, Query, Sort, StorageError, +}; +use tokio::sync::{Mutex, OnceCell}; + +use super::super::memory::MAX_GENERATIONS; +use crate::transcript::{ + SessionRef, SessionTranscript, TranscriptHistory, TranscriptLocator, TranscriptMessage, + TranscriptMeta, TranscriptPartial, TranscriptRead, TranscriptTurn, session_stem, +}; + +/// One document per transcript stem: the lookup fields. +pub(super) const INDEX: &str = "session_transcripts"; +/// One document per write to a transcript: its append-only log. +pub(super) const ENTRIES: &str = "session_transcript_entries"; + +/// Insert attempts before a contended write gives up. +const CAS_ATTEMPTS: usize = 64; + +/// How old an unwritten generation reservation must be before another +/// compaction may take it over. A reservation that old was left by a process +/// that stopped between reserving and writing; without a takeover its +/// sealed predecessor would refuse every later write. +const STALE_RESERVATION_MS: i64 = 30_000; + +/// Longest id before it is hashed, leaving room under the driver's limit for +/// an entry's `#` suffix. +const MAX_KEY_LEN: usize = 400; + +fn fnv1a(bytes: &[u8]) -> u64 { + bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) + }) +} + +/// A document id for `parts`: length-prefixed so no two tuples collide, and +/// hashed when it would exceed [`MAX_KEY_LEN`]. +pub(super) fn doc_key(parts: &[&str]) -> String { + let joined: String = parts + .iter() + .map(|part| format!("{}:{part}", part.len())) + .collect::>() + .join("/"); + if joined.len() <= MAX_KEY_LEN { + joined + } else { + format!("h:{:016x}:{}", fnv1a(joined.as_bytes()), joined.len()) + } +} + +fn entry_id(stem: &str, seq: u64) -> String { + format!("{}#{seq:010}", doc_key(&[stem])) +} + +/// Whether `stem` names a sub-agent transcript: `__` separates a parent stem +/// from its child's. +fn is_subagent(stem: &str) -> bool { + stem.split_once("__") + .is_some_and(|(parent, child)| !parent.is_empty() && !child.is_empty()) +} + +fn now_rfc3339() -> String { + chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Nanos, true) +} + +fn serialization(error: &serde_json::Error) -> StorageError { + StorageError::serialization(error.to_string()) +} + +/// A display-only partial as stored. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct StoredPartial { + content: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + iteration: Option, +} + +impl From<&TranscriptPartial> for StoredPartial { + fn from(partial: &TranscriptPartial) -> Self { + Self { + content: partial.content.clone(), + reasoning_content: partial.reasoning_content.clone(), + iteration: partial.iteration, + } + } +} + +impl From for TranscriptPartial { + fn from(stored: StoredPartial) -> Self { + Self { + content: stored.content, + reasoning_content: stored.reasoning_content, + iteration: stored.iteration, + } + } +} + +/// One entry of a transcript's log. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +struct Entry { + /// Whether this entry makes the transcript exist. Only a bare seal does + /// not: sealing a generation nobody wrote leaves it unwritten. + #[serde(default)] + written: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + meta: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + set: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + extend: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + tools: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + partial: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + request_id: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + clear: bool, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + seal: bool, +} + +/// A transcript rebuilt from its log, up to `next_seq`. +#[derive(Debug, Default)] +struct Replay { + next_seq: u64, + meta: Option, + messages: Vec, + tools: Option, + partials: Vec<(TranscriptPartial, Option)>, + written: bool, + sealed: bool, + /// The lookup fields last written to [`INDEX`] by this handle. + indexed: Option, +} + +impl Replay { + fn apply(&mut self, seq: u64, entry: Entry) { + self.next_seq = seq + 1; + if let Some(meta) = entry.meta { + self.meta = Some(meta); + } + if let Some(set) = entry.set { + self.messages = normalized(set); + } + if let Some(extend) = entry.extend { + self.messages.extend(normalized(extend)); + } + if let Some(tools) = entry.tools { + self.tools = Some(tools); + } + if let Some(partial) = entry.partial { + self.partials.push((partial.into(), entry.request_id)); + } + if entry.clear { + self.messages.clear(); + self.partials.clear(); + } + self.sealed |= entry.seal; + self.written |= entry.written; + } +} + +fn normalized(rows: Vec) -> Vec { + rows.into_iter().map(TranscriptMessage::normalized).collect() +} + +/// Declares both collections, once per locator. +#[derive(Debug, Default)] +struct Declared(OnceCell<()>); + +impl Declared { + async fn ensure(&self, docs: &Arc) -> Result<(), StorageError> { + self.0 + .get_or_try_init(|| async { + docs.ensure_collection( + &CollectionSpec::new(ENTRIES) + .index(IndexSpec::new("by_stem", ["stem", "seq"])) + .index(IndexSpec::new("by_stem_written", ["stem", "written"])), + ) + .await?; + docs.ensure_collection( + &CollectionSpec::new(INDEX) + .index(IndexSpec::new("by_thread", ["thread_id", "created_at"])) + .index(IndexSpec::new("by_agent_name", ["agent_name", "created_at"])) + .index(IndexSpec::new("by_agent_id", ["agent_id", "created_at"])), + ) + .await + }) + .await + .map(|_| ()) + } +} + +/// Runs `future` on `bridge`, flattening the bridge's own failure into the +/// driver error. +fn run_on(bridge: &Blocking, future: Fut) -> Result +where + Fut: Future> + Send + 'static, + T: Send + 'static, +{ + bridge.run(future)? +} + +// ── History ────────────────────────────────────────────────────────── + +/// A [`TranscriptHistory`] bound to one stem of a driver-backed locator. +/// +/// Handles are cheap and independent: two handles on one stem (or two +/// processes) stay consistent because every write claims the next log number +/// with an insert-only write and re-reads on a clash. +pub struct DriverTranscriptHistory { + inner: Arc, + bridge: Blocking, + path: PathBuf, +} + +struct HistoryInner { + docs: Arc, + declared: Arc, + stem: String, + seed: TranscriptMeta, + replay: Mutex, +} + +impl std::fmt::Debug for DriverTranscriptHistory { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DriverTranscriptHistory") + .field("path", &self.path) + .finish_non_exhaustive() + } +} + +impl DriverTranscriptHistory { + /// The display-only partials recorded so far, oldest first, with their + /// request ids. + /// + /// # Errors + /// + /// When the log cannot be read. + pub fn partials(&self) -> anyhow::Result)>> { + let inner = Arc::clone(&self.inner); + Ok(run_on(&self.bridge, async move { + let mut replay = inner.replay.lock().await; + inner.refresh(&mut replay).await?; + Ok(replay.partials.clone()) + })?) + } + + /// Appends `entry` built by `build` from the current replay; returns + /// whether anything was written (`build` answers `None` to write + /// nothing). + fn commit(&self, build: B) -> anyhow::Result + where + B: Fn(&Replay, &TranscriptMeta) -> anyhow::Result> + Send + 'static, + { + let inner = Arc::clone(&self.inner); + let outcome = run_on(&self.bridge, async move { + Ok(inner.commit(build).await) + })?; + outcome + } + + /// Seals this generation; later writes to it are refused. + fn seal(&self) -> anyhow::Result<()> { + self.commit(|replay, _| { + Ok((!replay.sealed).then(|| Entry { + seal: true, + ..Entry::default() + })) + }) + .map(|_| ()) + } + + /// Records the display-only `partial` of an interrupted turn, unless the + /// generation is sealed or the partial is empty. + fn record_partial( + &self, + partial: &TranscriptPartial, + request_id: Option<&str>, + ) -> anyhow::Result { + if partial.content.is_empty() { + return Ok(false); + } + let partial = StoredPartial::from(partial); + let request_id = request_id.map(str::to_string); + self.commit(move |replay, seed| { + Ok((!replay.sealed).then(|| Entry { + written: true, + meta: (!replay.written).then(|| seed.clone()), + partial: Some(partial.clone()), + request_id: request_id.clone(), + ..Entry::default() + })) + }) + } +} + +impl HistoryInner { + /// Applies every entry written since `replay` was last brought up to date. + async fn refresh(&self, replay: &mut Replay) -> Result<(), StorageError> { + self.declared.ensure(&self.docs).await?; + let query = Query::filter( + Filter::eq("stem", self.stem.as_str()).and(Filter::gte("seq", replay.next_seq)), + ) + .sort(Sort::asc("seq")) + .limit(500); + for stored in self.docs.query_all(ENTRIES, &query).await? { + let seq = stored.doc.get("seq").and_then(Value::as_u64).ok_or_else(|| { + StorageError::serialization(format!("transcript entry {} has no seq", stored.id)) + })?; + let entry: Entry = serde_json::from_value(stored.doc).map_err(|error| { + StorageError::serialization(format!( + "transcript entry {} is unreadable: {error}", + stored.id + )) + })?; + replay.apply(seq, entry); + } + Ok(()) + } + + async fn commit(&self, build: B) -> anyhow::Result + where + B: Fn(&Replay, &TranscriptMeta) -> anyhow::Result>, + { + let mut replay = self.replay.lock().await; + for _ in 0..CAS_ATTEMPTS { + self.refresh(&mut replay).await?; + let Some(entry) = build(&replay, &self.seed)? else { + return Ok(false); + }; + let seq = replay.next_seq; + let mut doc = serde_json::to_value(&entry).map_err(|error| serialization(&error))?; + if let Value::Object(fields) = &mut doc { + fields.insert("stem".into(), json!(self.stem)); + fields.insert("seq".into(), json!(seq)); + } + match self + .docs + .put(ENTRIES, &entry_id(&self.stem, seq), doc, Precondition::Absent) + .await + { + Ok(_) => { + replay.apply(seq, entry); + self.index(&mut replay).await?; + return Ok(true); + } + Err(error) if error.kind() == ErrorKind::Conflict => {} + Err(error) => return Err(error.into()), + } + } + anyhow::bail!( + "transcript {} kept changing under {CAS_ATTEMPTS} write attempts", + self.stem + ) + } + + /// Refreshes this stem's [`INDEX`] document when its lookup fields + /// changed. + async fn index(&self, replay: &mut Replay) -> Result<(), StorageError> { + if !replay.written { + return Ok(()); + } + let meta = replay.meta.as_ref().unwrap_or(&self.seed); + let fields = json!({ + "stem": self.stem, + "subagent": is_subagent(&self.stem), + "written": true, + "thread_id": meta.thread_id, + "agent_id": meta.agent_id, + "agent_name": meta.agent_name, + }); + if replay.indexed.as_ref() == Some(&fields) { + return Ok(()); + } + let id = doc_key(&[&self.stem]); + let created_at = self + .docs + .get(INDEX, &id) + .await? + .and_then(|found| found.doc.get("created_at").cloned()) + .unwrap_or_else(|| json!(now_rfc3339())); + let mut doc = fields.clone(); + doc["created_at"] = created_at; + self.docs.put(INDEX, &id, doc, Precondition::None).await?; + replay.indexed = Some(fields); + Ok(()) + } +} + +impl TranscriptRead for DriverTranscriptHistory { + fn path(&self) -> &Path { + &self.path + } + + fn read_session(&self) -> anyhow::Result> { + let inner = Arc::clone(&self.inner); + Ok(run_on(&self.bridge, async move { + let mut replay = inner.replay.lock().await; + inner.refresh(&mut replay).await?; + Ok(replay.written.then(|| SessionTranscript { + meta: replay.meta.clone().unwrap_or_else(|| inner.seed.clone()), + messages: replay.messages.clone(), + tools: replay.tools.clone(), + })) + })?) + } +} + +/// The entry recording a turn whose logical messages are now `next`: only +/// the new rows when `next` extends what is stored, all of them otherwise. +fn turn_entry(replay: &Replay, turn: &TurnRecord) -> anyhow::Result> { + anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); + let stored = &replay.messages; + let extends = replay.written + && turn.next.len() >= stored.len() + && turn.next[..stored.len()] == stored[..]; + let (set, extend) = if extends { + (None, Some(turn.next[stored.len()..].to_vec())) + } else { + (Some(turn.next.clone()), None) + }; + Ok(Some(Entry { + written: true, + meta: Some(turn.meta.clone()), + set, + extend, + tools: turn.tools.clone(), + partial: turn.partial.clone(), + request_id: turn.request_id.clone(), + ..Entry::default() + })) +} + +/// An owned [`TranscriptTurn`], so it can cross onto the bridge. +struct TurnRecord { + next: Vec, + meta: TranscriptMeta, + tools: Option, + request_id: Option, + partial: Option, +} + +impl TurnRecord { + fn new(turn: &TranscriptTurn<'_>, partial: Option<&TranscriptPartial>) -> Self { + Self { + next: normalized(turn.next.to_vec()), + meta: turn.meta.clone(), + tools: turn.tools.cloned(), + request_id: turn.request_id.map(str::to_string), + partial: partial + .filter(|partial| !partial.content.is_empty()) + .map(StoredPartial::from), + } + } +} + +impl TranscriptHistory for DriverTranscriptHistory { + fn append_turn(&self, turn: TranscriptTurn<'_>) -> anyhow::Result<()> { + let record = TurnRecord::new(&turn, None); + self.commit(move |replay, _| turn_entry(replay, &record)) + .map(|_| ()) + } + + fn append_turn_with_partial( + &self, + turn: TranscriptTurn<'_>, + partial: Option<&TranscriptPartial>, + ) -> anyhow::Result<()> { + let record = TurnRecord::new(&turn, partial); + self.commit(move |replay, _| turn_entry(replay, &record)) + .map(|_| ()) + } + + fn messages(&self) -> anyhow::Result> { + let inner = Arc::clone(&self.inner); + Ok(run_on(&self.bridge, async move { + let mut replay = inner.replay.lock().await; + inner.refresh(&mut replay).await?; + Ok(replay.messages.clone()) + })?) + } + + fn append(&self, message: TranscriptMessage) -> anyhow::Result<()> { + let message = message.normalized(); + self.commit(move |replay, seed| { + anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); + Ok(Some(Entry { + written: true, + meta: (!replay.written).then(|| seed.clone()), + extend: Some(vec![message.clone()]), + ..Entry::default() + })) + }) + .map(|_| ()) + } + + fn replace(&self, messages: &[TranscriptMessage]) -> anyhow::Result<()> { + let messages = normalized(messages.to_vec()); + self.commit(move |replay, seed| { + anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); + Ok(Some(Entry { + written: true, + meta: (!replay.written).then(|| seed.clone()), + set: Some(messages.clone()), + ..Entry::default() + })) + }) + .map(|_| ()) + } + + fn clear(&self) -> anyhow::Result<()> { + self.commit(|replay, _| { + anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); + Ok(replay.written.then(|| Entry { + written: true, + clear: true, + ..Entry::default() + })) + }) + .map(|_| ()) + } +} + +// ── Locator ────────────────────────────────────────────────────────── + +/// A [`TranscriptLocator`] keeping transcripts in a driver +/// [`DocumentStore`]; see the module docs for the layout. +#[derive(Clone)] +pub struct DriverTranscriptLocator { + docs: Arc, + bridge: Blocking, + label: String, + declared: Arc, +} + +impl std::fmt::Debug for DriverTranscriptLocator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DriverTranscriptLocator") + .field("label", &self.label) + .finish_non_exhaustive() + } +} + +impl DriverTranscriptLocator { + /// Transcripts in `docs`, with synchronous calls run on `bridge`. + /// + /// `label` names where `docs` points (driver, database, scope). It is + /// the locator's [`destination_key`](TranscriptLocator::destination_key) + /// and the prefix of every handle's path, so two locators should share a + /// label only when they address the same data. + pub fn new(docs: Arc, bridge: Blocking, label: impl Into) -> Self { + Self { + docs, + bridge, + label: label.into(), + declared: Arc::new(Declared::default()), + } + } + + fn handle(&self, stem: &str, seed: TranscriptMeta) -> DriverTranscriptHistory { + DriverTranscriptHistory { + inner: Arc::new(HistoryInner { + docs: Arc::clone(&self.docs), + declared: Arc::clone(&self.declared), + stem: stem.to_string(), + seed, + replay: Mutex::new(Replay::default()), + }), + bridge: self.bridge.clone(), + path: PathBuf::from(format!("{}/{stem}", self.label)), + } + } + + fn run(&self, op: F) -> Result + where + F: FnOnce(Arc, Arc) -> Fut, + Fut: Future> + Send + 'static, + T: Send + 'static, + { + run_on( + &self.bridge, + op(Arc::clone(&self.docs), Arc::clone(&self.declared)), + ) + } + + /// The newest written root transcript matching `filter`. + fn newest_root(&self, filter: Filter, what: &str) -> Option> { + let found = self.run(|docs, declared| async move { + declared.ensure(&docs).await?; + let query = Query::filter( + filter + .and(Filter::eq("subagent", false)) + .and(Filter::eq("written", true)), + ) + .sort(Sort::desc("created_at")) + .limit(1); + Ok(docs + .query(INDEX, &query) + .await? + .items + .into_iter() + .next() + .and_then(|found| found.doc.get("stem").and_then(Value::as_str).map(str::to_string))) + }); + match found { + Ok(stem) => stem.map(|stem| { + let seed = discovered_seed(&stem); + Arc::new(self.handle(&stem, seed)) as Arc + }), + Err(error) => { + tracing::warn!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + lookup = what, + %error, + "[session-store] transcript lookup failed" + ); + None + } + } + } + + /// Reserves `stem` as a new generation: succeeds when nothing is written + /// there and no live reservation holds it. + async fn reserve( + docs: &Arc, + stem: &str, + ) -> anyhow::Result<()> { + let written = docs + .count( + ENTRIES, + &Filter::eq("stem", stem).and(Filter::eq("written", true)), + ) + .await?; + anyhow::ensure!(written == 0, "session generation already exists"); + let id = doc_key(&[stem]); + let now = chrono::Utc::now().timestamp_millis(); + let reservation = json!({ + "stem": stem, + "subagent": is_subagent(stem), + "written": false, + "created_at": now_rfc3339(), + "reserved_at": now, + }); + let precondition = match docs.get(INDEX, &id).await? { + None => Precondition::Absent, + Some(found) => { + let stale = found.doc.get("written") == Some(&json!(false)) + && found + .doc + .get("reserved_at") + .and_then(Value::as_i64) + .is_some_and(|at| now - at >= STALE_RESERVATION_MS); + anyhow::ensure!(stale, "session generation already exists or is reserved"); + found.unchanged() + } + }; + match docs.put(INDEX, &id, reservation, precondition).await { + Ok(_) => Ok(()), + Err(error) if error.kind() == ErrorKind::Conflict => { + anyhow::bail!("session generation already exists or is reserved") + } + Err(error) => Err(error.into()), + } + } +} + +/// A seed for a handle on a transcript found by lookup. It is used only if +/// the transcript turns out to be unwritten, which a lookup never returns. +fn discovered_seed(stem: &str) -> TranscriptMeta { + TranscriptMeta { + agent_name: stem.to_string(), + agent_id: None, + agent_type: None, + dispatcher: String::new(), + provider: None, + model: None, + created: String::new(), + updated: String::new(), + turn_count: 0, + prefix_message_count: None, + input_tokens: 0, + output_tokens: 0, + cached_input_tokens: 0, + charged_amount_usd: 0.0, + thread_id: None, + task_id: None, + session_id: None, + parent_session_id: None, + } +} + +impl TranscriptLocator for DriverTranscriptLocator { + fn destination_key(&self) -> Option { + Some(format!("{}/transcripts", self.label)) + } + + fn latest_for_agent(&self, agent_name: &str) -> Option> { + let filter = Filter::eq("agent_name", agent_name).or(Filter::eq("agent_id", agent_name)); + self.newest_root(filter, "latest_for_agent") + } + + fn root_for_thread(&self, thread_id: &str) -> Option> { + self.root_for_thread_scoped(thread_id, None) + } + + fn root_for_thread_scoped( + &self, + thread_id: &str, + agent_id: Option<&str>, + ) -> Option> { + let thread_id = thread_id.trim(); + if thread_id.is_empty() { + return None; + } + let mut filter = Filter::eq("thread_id", thread_id); + if let Some(agent_id) = agent_id { + filter = filter.and(Filter::eq("agent_id", agent_id)); + } + self.newest_root(filter, "root_for_thread") + } + + fn open_stem( + &self, + stem: &str, + seed: TranscriptMeta, + ) -> anyhow::Result> { + Ok(Arc::new(self.handle(stem, seed))) + } + + fn session_exists(&self, session: &SessionRef) -> bool { + let stem = session_stem(session); + let written = self.run(|docs, declared| async move { + declared.ensure(&docs).await?; + docs.count( + ENTRIES, + &Filter::eq("stem", stem).and(Filter::eq("written", true)), + ) + .await + }); + match written { + Ok(count) => count > 0, + Err(error) => { + tracing::warn!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + %error, + "[session-store] transcript existence check failed" + ); + false + } + } + } + + fn read_session_transcript(&self, session: &SessionRef) -> Option> { + if !self.session_exists(session) { + return None; + } + let stem = session_stem(session); + let seed = discovered_seed(&stem); + Some(Arc::new(self.handle(&stem, seed))) + } + + fn append_interrupted_partial( + &self, + thread_id: &str, + agent_id: Option<&str>, + partial: &TranscriptPartial, + request_id: Option<&str>, + ) -> anyhow::Result { + if partial.content.is_empty() { + return Ok(false); + } + let thread_id = thread_id.trim(); + if thread_id.is_empty() { + return Ok(false); + } + let mut filter = Filter::eq("thread_id", thread_id); + if let Some(agent_id) = agent_id { + filter = filter.and(Filter::eq("agent_id", agent_id)); + } + let Some(root) = self.newest_root(filter, "append_interrupted_partial") else { + return Ok(false); + }; + let stem = root + .path() + .strip_prefix(&self.label) + .ok() + .and_then(Path::to_str) + .map(str::to_string) + .ok_or_else(|| anyhow::anyhow!("transcript handle outside this locator"))?; + self.handle(&stem, discovered_seed(&stem)) + .record_partial(partial, request_id) + } + + /// Reserves the successor generation, then seals `session`. The + /// reservation is what keeps two compactions from both opening the + /// successor; it is released again if the seal fails. + fn begin_generation( + &self, + session: &SessionRef, + seed: TranscriptMeta, + ) -> anyhow::Result<(SessionRef, Arc)> { + let successor = session.next_generation(); + anyhow::ensure!( + successor.generation <= MAX_GENERATIONS, + "session generation limit reached" + ); + let stem = session_stem(&successor); + let reserved = { + let stem = stem.clone(); + run_on(&self.bridge, { + let docs = Arc::clone(&self.docs); + let declared = Arc::clone(&self.declared); + async move { + declared.ensure(&docs).await?; + Ok(Self::reserve(&docs, &stem).await) + } + })? + }; + reserved?; + let predecessor_stem = session_stem(session); + if let Err(error) = self + .handle(&predecessor_stem, seed.clone()) + .seal() + { + let docs = Arc::clone(&self.docs); + let id = doc_key(&[&stem]); + if let Err(release) = run_on(&self.bridge, async move { + docs.delete(INDEX, &id, Precondition::None).await + }) { + tracing::warn!( + target: "tinyagents_session::port::drivers", + stem = %stem, + error = %release, + "[session-store] could not release a generation reservation" + ); + } + return Err(error); + } + let mut meta = seed; + meta.session_id = Some(successor.session_id()); + meta.parent_session_id = successor.parent_session_id(); + let handle = self.handle(&stem, meta); + Ok((successor, Arc::new(handle))) + } +} + +#[cfg(test)] +#[path = "transcripts_tests.rs"] +mod tests; From ced07db4cfd0d2e2d86c6ebd231a89557cd72920 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:23:51 +0300 Subject: [PATCH 20/56] feat(session): expose storage-driver session stores behind a feature The storage-driver session stores are now re-exported from the crate root and the port module when the `storage-drivers` feature is enabled, so hosts can reach them without depending on internal paths. Transcript lookup was also split so the newest root stem can be resolved without building a handle, letting interrupted-partial appends reuse the stem directly. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 2 + crates/tinyagents-session/src/lib.rs | 4 ++ .../src/port/drivers/mod_tests.rs | 0 .../src/port/drivers/transcripts.rs | 55 ++++++++++--------- .../src/port/drivers/transcripts_tests.rs | 0 .../src/port/drivers/turn_states_tests.rs | 0 crates/tinyagents-session/src/port/mod.rs | 9 +++ 7 files changed, 43 insertions(+), 27 deletions(-) create mode 100644 crates/tinyagents-session/src/port/drivers/mod_tests.rs create mode 100644 crates/tinyagents-session/src/port/drivers/transcripts_tests.rs create mode 100644 crates/tinyagents-session/src/port/drivers/turn_states_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1a7a1eed7..df7d8679d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1886,6 +1886,7 @@ dependencies = [ "tempfile", "thiserror", "tinyagents-harness", + "tinystoragedrivers-core", "tinytools-agent", "tokio", "tracing", @@ -1992,6 +1993,7 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tokio", ] [[package]] diff --git a/crates/tinyagents-session/src/lib.rs b/crates/tinyagents-session/src/lib.rs index 07c094eee..8fe85817f 100644 --- a/crates/tinyagents-session/src/lib.rs +++ b/crates/tinyagents-session/src/lib.rs @@ -108,6 +108,10 @@ pub use ops::{ record_message_with_reasoning, record_session_end, record_session_start, record_tool_call, search_sessions, set_fts_snippet_bytes, }; +#[cfg(feature = "storage-drivers")] +pub use port::{ + DriverSessionStores, DriverTranscriptHistory, DriverTranscriptLocator, DriverTurnStates, +}; pub use port::{ AgentStores, InMemorySessionStores, InMemoryTranscriptLocator, InMemoryTurnStates, SessionStoreProvider, TurnStates, diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs new file mode 100644 index 000000000..e69de29bb diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index 900b6c400..f9111a01e 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -622,8 +622,8 @@ impl DriverTranscriptLocator { ) } - /// The newest written root transcript matching `filter`. - fn newest_root(&self, filter: Filter, what: &str) -> Option> { + /// The stem of the newest written root transcript matching `filter`. + fn newest_root_stem(&self, filter: Filter, what: &str) -> Option { let found = self.run(|docs, declared| async move { declared.ensure(&docs).await?; let query = Query::filter( @@ -639,24 +639,32 @@ impl DriverTranscriptLocator { .items .into_iter() .next() - .and_then(|found| found.doc.get("stem").and_then(Value::as_str).map(str::to_string))) + .and_then(|found| { + found + .doc + .get("stem") + .and_then(Value::as_str) + .map(str::to_string) + })) }); - match found { - Ok(stem) => stem.map(|stem| { - let seed = discovered_seed(&stem); - Arc::new(self.handle(&stem, seed)) as Arc - }), - Err(error) => { - tracing::warn!( - target: "tinyagents_session::port::drivers", - locator = %self.label, - lookup = what, - %error, - "[session-store] transcript lookup failed" - ); - None - } - } + found.unwrap_or_else(|error| { + tracing::warn!( + target: "tinyagents_session::port::drivers", + locator = %self.label, + lookup = what, + %error, + "[session-store] transcript lookup failed" + ); + None + }) + } + + /// The newest written root transcript matching `filter`. + fn newest_root(&self, filter: Filter, what: &str) -> Option> { + self.newest_root_stem(filter, what).map(|stem| { + let seed = discovered_seed(&stem); + Arc::new(self.handle(&stem, seed)) as Arc + }) } /// Reserves `stem` as a new generation: succeeds when nothing is written @@ -818,16 +826,9 @@ impl TranscriptLocator for DriverTranscriptLocator { if let Some(agent_id) = agent_id { filter = filter.and(Filter::eq("agent_id", agent_id)); } - let Some(root) = self.newest_root(filter, "append_interrupted_partial") else { + let Some(stem) = self.newest_root_stem(filter, "append_interrupted_partial") else { return Ok(false); }; - let stem = root - .path() - .strip_prefix(&self.label) - .ok() - .and_then(Path::to_str) - .map(str::to_string) - .ok_or_else(|| anyhow::anyhow!("transcript handle outside this locator"))?; self.handle(&stem, discovered_seed(&stem)) .record_partial(partial, request_id) } diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs new file mode 100644 index 000000000..e69de29bb diff --git a/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs b/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs new file mode 100644 index 000000000..e69de29bb diff --git a/crates/tinyagents-session/src/port/mod.rs b/crates/tinyagents-session/src/port/mod.rs index 94c3bdd62..0d390e12a 100644 --- a/crates/tinyagents-session/src/port/mod.rs +++ b/crates/tinyagents-session/src/port/mod.rs @@ -34,15 +34,24 @@ //! //! - [`InMemorySessionStores`]: per-agent, process-lifetime storage, for //! tests and for hosts that deliberately keep nothing. +//! - `DriverSessionStores` (feature `storage-drivers`): every agent's stores +//! in one `tinystoragedrivers` backend (SQLite, MongoDB, ...), each agent +//! under its own storage scope. //! - The on-disk layout (`session_raw/`, `tinyagents_store/`, turn-state //! files) is wrapped by the host that owns it, not here: this crate keeps //! the file and SQLite building blocks, and a host crate decides to use them. +#[cfg(feature = "storage-drivers")] +mod drivers; mod memory; mod types; use std::sync::Arc; +#[cfg(feature = "storage-drivers")] +pub use drivers::{ + DriverSessionStores, DriverTranscriptHistory, DriverTranscriptLocator, DriverTurnStates, +}; pub use memory::{InMemorySessionStores, InMemoryTranscriptLocator, InMemoryTurnStates}; pub use types::AgentStores; From 4a897c6be7df6f7efc9c0d742aa0552976c019f3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:24:15 +0300 Subject: [PATCH 21/56] chore(session): add sqlite dev-dependency for storage driver tests Adds the tinystoragedrivers-sqlite crate as a dev-dependency so the storage-drivers provider can be exercised against a real SQLite backend in tests. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/tinyagents-session/Cargo.toml b/crates/tinyagents-session/Cargo.toml index 180c5ec01..eff5b34d9 100644 --- a/crates/tinyagents-session/Cargo.toml +++ b/crates/tinyagents-session/Cargo.toml @@ -46,6 +46,8 @@ storage-drivers = [ ] [dev-dependencies] +# The `storage-drivers` provider is checked against a real SQLite backend too. +tinystoragedrivers-sqlite = { path = "../../vendor/tinystoragedrivers/crates/tinystoragedrivers-sqlite" } tokio = { workspace = true, default-features = true, features = ["macros", "rt-multi-thread"] } [lints] From 0cddb45b5598efc7ac334378bf06610bcaa90b81 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:24:41 +0300 Subject: [PATCH 22/56] test(session): add tests for session port drivers Add unit tests covering the session port driver behaviour so the module's contract is exercised directly. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/mod_tests.rs | 222 ++++++++++++++++++ 1 file changed, 222 insertions(+) diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs index e69de29bb..60a864d9e 100644 --- a/crates/tinyagents-session/src/port/drivers/mod_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/mod_tests.rs @@ -0,0 +1,222 @@ +use super::*; +use crate::testkit::conformance::{session_store_conformance, session_store_isolation_conformance}; +use crate::turn_state::{TurnLifecycle, TurnState}; +use tinystoragedrivers_core::MemoryStorage; +use tinystoragedrivers_sqlite::SqliteStorage; + +fn memory() -> DriverSessionStores { + DriverSessionStores::new(Arc::new(MemoryStorage::new())).unwrap() +} + +fn sqlite(dir: &tempfile::TempDir) -> DriverSessionStores { + let storage = SqliteStorage::open(dir.path().join("sessions.db")).unwrap(); + DriverSessionStores::new(Arc::new(storage)).unwrap() +} + +#[tokio::test] +async fn memory_backed_stores_meet_the_session_store_contract() { + session_store_conformance(&memory()).await; +} + +#[tokio::test] +async fn memory_backed_stores_keep_agents_apart() { + session_store_isolation_conformance(&memory()).await; +} + +#[tokio::test] +async fn sqlite_backed_stores_meet_the_session_store_contract() { + let dir = tempfile::tempdir().unwrap(); + session_store_conformance(&sqlite(&dir)).await; +} + +#[tokio::test] +async fn sqlite_backed_stores_keep_agents_apart() { + let dir = tempfile::tempdir().unwrap(); + session_store_isolation_conformance(&sqlite(&dir)).await; +} + +#[tokio::test(flavor = "multi_thread")] +async fn stores_work_from_a_multi_threaded_runtime() { + session_store_conformance(&memory()).await; +} + +#[test] +fn stores_work_outside_any_runtime() { + let stores = memory().for_agent("plain"); + let turn = TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z"); + stores.turn_states.put(&turn).unwrap(); + assert_eq!(stores.turn_states.get("t").unwrap(), Some(turn)); +} + +#[test] +fn a_valid_agent_id_is_its_own_scope() { + assert_eq!(DriverSessionStores::scope_for("agent-7").as_str(), "agent-7"); +} + +#[test] +fn any_other_agent_id_maps_to_a_stable_hashed_scope() { + let spaced = DriverSessionStores::scope_for("has a space"); + assert!(spaced.as_str().starts_with("sha256:")); + assert_eq!(spaced, DriverSessionStores::scope_for("has a space")); + assert_ne!(spaced, DriverSessionStores::scope_for("has a space")); + assert!(DriverSessionStores::scope_for("").as_str().starts_with("sha256:")); + let long = "a".repeat(300); + assert!(DriverSessionStores::scope_for(&long).as_str().starts_with("sha256:")); +} + +#[test] +fn an_agent_gets_the_same_stores_every_time() { + let provider = memory(); + let first = provider.for_agent("a"); + let again = provider.for_agent("a"); + assert!(Arc::ptr_eq(&first.turn_states, &again.turn_states)); + assert_eq!( + first.transcripts.destination_key(), + again.transcripts.destination_key() + ); + assert_ne!( + first.transcripts.destination_key(), + provider.for_agent("b").transcripts.destination_key() + ); + let debug = format!("{provider:?}"); + assert!(debug.contains("agents: 2"), "{debug}"); + assert!(debug.contains("memory"), "{debug}"); + assert!( + provider + .destination_key() + .is_some_and(|key| key.starts_with("memory://")) + ); +} + +#[test] +fn data_survives_a_new_provider_on_the_same_backend() { + let backend: Arc = Arc::new(MemoryStorage::new()); + let turn = TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z"); + DriverSessionStores::new(Arc::clone(&backend)) + .unwrap() + .for_agent("a") + .turn_states + .put(&turn) + .unwrap(); + let reopened = DriverSessionStores::new(backend).unwrap().for_agent("a"); + assert_eq!(reopened.turn_states.get("t").unwrap(), Some(turn)); +} + +#[test] +fn recover_marks_open_agents_turns_interrupted() { + let provider = memory(); + let stores = provider.for_agent("a"); + stores + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .unwrap(); + provider.recover().unwrap(); + assert_eq!( + stores.turn_states.get("t").unwrap().unwrap().lifecycle, + TurnLifecycle::Interrupted + ); +} + +#[test] +fn recover_on_open_interrupts_turns_an_earlier_process_left() { + let backend: Arc = Arc::new(MemoryStorage::new()); + let earlier = DriverSessionStores::new(Arc::clone(&backend)).unwrap(); + earlier + .for_agent("a") + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .unwrap(); + + let plain = DriverSessionStores::new(Arc::clone(&backend)).unwrap(); + assert_eq!( + plain.for_agent("a").turn_states.get("t").unwrap().unwrap().lifecycle, + TurnLifecycle::Started, + "without the option nothing is touched" + ); + + let recovering = DriverSessionStores::new(backend) + .unwrap() + .recover_on_open(true); + assert!(format!("{recovering:?}").contains("recover_on_open: true")); + let stores = recovering.for_agent("a"); + assert_eq!( + stores.turn_states.get("t").unwrap().unwrap().lifecycle, + TurnLifecycle::Interrupted + ); + + // Only the first open recovers: a turn started afterwards stays live. + stores + .turn_states + .put(&TurnState::started("t2", "r", 8, "2026-01-01T00:01:00Z")) + .unwrap(); + recovering.agents.lock().unwrap().clear(); + assert_eq!( + recovering + .for_agent("a") + .turn_states + .get("t2") + .unwrap() + .unwrap() + .lifecycle, + TurnLifecycle::Started + ); +} + +/// A backend that cannot bind any scope. +#[derive(Debug)] +struct Unbindable; + +impl StorageBackend for Unbindable { + fn driver(&self) -> &'static str { + "unbindable" + } + + fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + tinystoragedrivers_core::Capabilities::default() + } + + fn for_scope(&self, _scope: &Scope) -> Result { + Err(StorageError::unavailable("database is down")) + } + + fn database(&self, _name: &str) -> Result, StorageError> { + Err(StorageError::unavailable("database is down")) + } +} + +#[tokio::test] +async fn an_unbindable_scope_fails_closed() { + let provider = DriverSessionStores::new(Arc::new(Unbindable)).unwrap(); + assert!(provider.try_for_agent("a").is_err()); + let stores = provider.for_agent("a"); + + let error = stores + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .unwrap_err(); + assert!(error.contains("database is down"), "{error}"); + assert!(stores.turn_states.list().is_err()); + assert!(stores.kv.get("ns", "k").await.is_err()); + assert!(stores.journal.append("s", serde_json::json!(1)).await.is_err()); + assert!(stores.journal.len("s").await.is_err()); + + let session = crate::transcript::SessionRef::scoped("t", "a"); + assert!(!stores.transcripts.session_exists(&session)); + assert!(stores.transcripts.root_for_thread("t").is_none()); + assert!(stores.transcripts.latest_for_agent("a").is_none()); + let handle = stores + .transcripts + .open_stem("stem", transcripts::tests::meta("t")) + .unwrap(); + assert!(handle.read_session().is_err()); + assert!(handle.messages().is_err()); + assert!( + stores + .transcripts + .begin_generation(&session, transcripts::tests::meta("t")) + .is_err() + ); + + assert!(provider.agents.lock().unwrap().is_empty(), "refusals are not cached"); + provider.recover().unwrap(); +} From 09dd24fb45ba2c5fcaaaacb53f78bc681263ba10 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:24:49 +0300 Subject: [PATCH 23/56] fix(session): correct refused driver test expectations The refused driver tests were asserting the wrong error variants and message text, so they passed against behaviour the driver no longer produces. The expectations now match what the driver actually returns. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/refused_tests.rs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 crates/tinyagents-session/src/port/drivers/refused_tests.rs diff --git a/crates/tinyagents-session/src/port/drivers/refused_tests.rs b/crates/tinyagents-session/src/port/drivers/refused_tests.rs new file mode 100644 index 000000000..119cfeee7 --- /dev/null +++ b/crates/tinyagents-session/src/port/drivers/refused_tests.rs @@ -0,0 +1,50 @@ +use super::*; + +fn refused() -> Refused { + Refused { + kind: ErrorKind::Unavailable, + message: "down".into(), + } +} + +#[tokio::test] +async fn every_document_call_reports_the_refusal() { + let docs = refused(); + let kind = |error: StorageError| { + assert_eq!(error.message(), "down"); + error.kind() + }; + assert_eq!(docs.capabilities(), Capabilities::default()); + assert_eq!( + kind(docs.ensure_collection(&CollectionSpec::new("c")).await.unwrap_err()), + ErrorKind::Unavailable + ); + assert!(docs.get("c", "i").await.is_err()); + assert!( + docs.put("c", "i", Value::Null, Precondition::None) + .await + .is_err() + ); + assert!(docs.delete("c", "i", Precondition::None).await.is_err()); + assert!(docs.query("c", &Query::all()).await.is_err()); + assert!(docs.count("c", &Filter::All).await.is_err()); + assert!(docs.delete_where("c", &Filter::All).await.is_err()); + assert!( + docs.claim("c", &Filter::All, &[], &Value::Null) + .await + .is_err() + ); + assert!(docs.drop_collection("c").await.is_err()); +} + +#[tokio::test] +async fn every_stream_call_reports_the_refusal() { + let streams = refused(); + assert!(streams.append("s", Value::Null).await.is_err()); + assert!(streams.append_batch("s", vec![]).await.is_err()); + assert!(streams.read_window("s", 0, 1).await.is_err()); + assert!(StreamStore::len(&streams, "s").await.is_err()); + assert!(streams.truncate_before("s", 0).await.is_err()); + assert!(streams.delete_stream("s").await.is_err()); + assert!(streams.streams("").await.is_err()); +} From da3b15ce7f6897bfa4bb4531c843b00932c305be Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:24:56 +0300 Subject: [PATCH 24/56] refactor(session): extract refused driver into its own module Moved the refused driver out of the transcripts module into a dedicated refused module so each driver lives in its own file. No behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/port/drivers/refused.rs | 4 ++++ crates/tinyagents-session/src/port/drivers/transcripts.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-session/src/port/drivers/refused.rs b/crates/tinyagents-session/src/port/drivers/refused.rs index 3d96b3ad6..f6bfb3aa9 100644 --- a/crates/tinyagents-session/src/port/drivers/refused.rs +++ b/crates/tinyagents-session/src/port/drivers/refused.rs @@ -131,3 +131,7 @@ impl StreamStore for Refused { Err(self.error()) } } + +#[cfg(test)] +#[path = "refused_tests.rs"] +mod tests; diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index f9111a01e..ff6ea71c6 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -888,4 +888,4 @@ impl TranscriptLocator for DriverTranscriptLocator { #[cfg(test)] #[path = "transcripts_tests.rs"] -mod tests; +pub(super) mod tests; From a9996d634f1d9785f1858cd8923ef409491de526 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:25:43 +0300 Subject: [PATCH 25/56] test(session): cover transcript driver edge cases Add tests for the transcript port driver covering empty transcripts, missing entries, and round-trip persistence so regressions in the session storage layer are caught early. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts_tests.rs | 443 ++++++++++++++++++ 1 file changed, 443 insertions(+) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index e69de29bb..ddf79eb4f 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -0,0 +1,443 @@ +use super::*; +use crate::testkit::conformance::transcript_history_conformance; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend, Version}; + +pub(in crate::port::drivers) fn meta(thread_id: &str) -> TranscriptMeta { + let mut meta = discovered_seed("planner"); + meta.agent_id = Some("planner".to_string()); + meta.dispatcher = "native".to_string(); + meta.created = "2026-01-01T00:00:00Z".to_string(); + meta.updated = "2026-01-01T00:00:00Z".to_string(); + meta.thread_id = Some(thread_id.to_string()); + meta +} + +fn message(role: &str, content: &str) -> TranscriptMessage { + serde_json::from_value(json!({ "role": role, "content": content })).unwrap() +} + +fn docs() -> Arc { + let storage = MemoryStorage::new(); + Arc::clone(storage.for_scope(&Scope::local()).unwrap().documents()) +} + +fn locator(docs: &Arc) -> DriverTranscriptLocator { + DriverTranscriptLocator::new(Arc::clone(docs), Blocking::new().unwrap(), "memory://test") +} + +fn turn<'a>( + prev: &'a [TranscriptMessage], + next: &'a [TranscriptMessage], + meta: &'a TranscriptMeta, +) -> TranscriptTurn<'a> { + TranscriptTurn { + prev, + next, + meta, + turn_usage: None, + request_id: Some("req"), + tools: None, + } +} + +/// The raw log entries of `stem`, in order. +fn entries(docs: &Arc, stem: &str) -> Vec { + let docs = Arc::clone(docs); + let stem = stem.to_string(); + Blocking::new() + .unwrap() + .run(async move { + docs.query_all( + ENTRIES, + &Query::filter(Filter::eq("stem", stem)).sort(Sort::asc("seq")), + ) + .await + .unwrap() + .into_iter() + .map(|found| found.doc) + .collect() + }) + .unwrap() +} + +fn on_bridge( + future: impl Future> + Send + 'static, +) -> T { + Blocking::new().unwrap().run(future).unwrap().unwrap() +} + +#[test] +fn a_handle_meets_the_transcript_history_contract() { + let docs = docs(); + let history = locator(&docs).open_stem("contract", meta("t")).unwrap(); + transcript_history_conformance(history.as_ref()); +} + +#[test] +fn an_ordinary_turn_stores_only_its_new_rows() { + let docs = docs(); + let history = locator(&docs).open_stem("s", meta("t")).unwrap(); + let first = vec![message("user", "a"), message("assistant", "b")]; + history.append_turn(turn(&[], &first, &meta("t"))).unwrap(); + let mut second = first.clone(); + second.extend([message("user", "c"), message("assistant", "d")]); + history.append_turn(turn(&first, &second, &meta("t"))).unwrap(); + let compacted = vec![message("user", "summary")]; + history + .append_turn(turn(&second, &compacted, &meta("t"))) + .unwrap(); + + let log = entries(&docs, "s"); + assert_eq!(log.len(), 3); + assert_eq!(log[0]["set"].as_array().unwrap().len(), 2); + assert_eq!(log[1]["extend"].as_array().unwrap().len(), 2); + assert!(log[1].get("set").is_none()); + assert_eq!(log[2]["set"].as_array().unwrap().len(), 1); + assert_eq!(log[2]["seq"], json!(2)); + assert_eq!(history.messages().unwrap(), normalized(compacted)); +} + +#[test] +fn two_handles_on_one_stem_never_lose_each_others_writes() { + let docs = docs(); + let locator = locator(&docs); + let one = locator.open_stem("s", meta("t")).unwrap(); + let two = locator.open_stem("s", meta("t")).unwrap(); + one.append(message("user", "from one")).unwrap(); + two.append(message("user", "from two")).unwrap(); + one.append(message("user", "one again")).unwrap(); + let contents: Vec = two + .messages() + .unwrap() + .into_iter() + .map(|row| row.content) + .collect(); + assert_eq!(contents, ["from one", "from two", "one again"]); +} + +#[test] +fn concurrent_writers_serialize_through_the_log() { + let docs = docs(); + let locator = locator(&docs); + let threads: Vec<_> = (0..4) + .map(|writer| { + let handle = locator.open_stem("busy", meta("t")).unwrap(); + std::thread::spawn(move || { + for i in 0..5 { + handle + .append(message("user", &format!("{writer}-{i}"))) + .unwrap(); + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap(); + } + let reader = locator.open_stem("busy", meta("t")).unwrap(); + assert_eq!(reader.messages().unwrap().len(), 20); + let seqs: Vec = entries(&docs, "busy") + .iter() + .map(|entry| entry["seq"].as_u64().unwrap()) + .collect(); + assert_eq!(seqs, (0..20).collect::>(), "the log is dense"); +} + +#[test] +fn tools_and_meta_follow_the_latest_turn() { + let docs = docs(); + let history = locator(&docs).open_stem("s", meta("t")).unwrap(); + let rows = vec![message("user", "a")]; + let tools = json!([{ "name": "search" }]); + history + .append_turn(TranscriptTurn { + tools: Some(&tools), + ..turn(&[], &rows, &meta("t")) + }) + .unwrap(); + let mut later = meta("t"); + later.turn_count = 2; + history.append_turn(turn(&rows, &rows, &later)).unwrap(); + let read = history.read_session().unwrap().unwrap(); + assert_eq!(read.tools, Some(tools), "a turn without tools keeps the last"); + assert_eq!(read.meta.turn_count, 2); +} + +#[test] +fn partials_stay_out_of_the_replay_and_a_clear_drops_them() { + let docs = docs(); + let locator = locator(&docs); + let handle = locator.handle("s", meta("t")); + assert!(!handle.record_partial(&TranscriptPartial::new(""), None).unwrap()); + let rows = vec![message("user", "a")]; + let mut partial = TranscriptPartial::new("half"); + partial.reasoning_content = Some("thinking".into()); + partial.iteration = Some(2); + handle + .append_turn_with_partial(turn(&[], &rows, &meta("t")), Some(&partial)) + .unwrap(); + assert!(handle.record_partial(&TranscriptPartial::new("more"), Some("r2")).unwrap()); + let partials = handle.partials().unwrap(); + assert_eq!(partials.len(), 2); + assert_eq!(partials[0], (partial, Some("req".to_string()))); + assert_eq!(partials[1].1.as_deref(), Some("r2")); + assert_eq!(handle.messages().unwrap().len(), 1); + handle.clear().unwrap(); + assert!(handle.partials().unwrap().is_empty()); + assert!(handle.messages().unwrap().is_empty()); + assert!( + handle.read_session().unwrap().is_some(), + "a cleared transcript still exists" + ); +} + +#[test] +fn a_partial_alone_makes_the_transcript_exist() { + let docs = docs(); + let handle = locator(&docs).handle("s", meta("t")); + assert!(handle.record_partial(&TranscriptPartial::new("half"), None).unwrap()); + let read = handle.read_session().unwrap().unwrap(); + assert!(read.messages.is_empty()); + assert_eq!(read.meta.thread_id.as_deref(), Some("t"), "the seed is the meta"); +} + +#[test] +fn a_sealed_generation_refuses_writes() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + let history = locator.open_session(&session, meta("t")).unwrap(); + history.append(message("user", "a")).unwrap(); + let (successor, _next) = locator.begin_generation(&session, meta("t")).unwrap(); + assert_eq!(successor.generation, 1); + for error in [ + history.append(message("user", "late")).unwrap_err(), + history.replace(&[]).unwrap_err(), + history.clear().unwrap_err(), + history + .append_turn(turn(&[], &[], &meta("t"))) + .unwrap_err(), + ] { + assert!(error.to_string().contains("sealed"), "{error}"); + } + let handle = locator.handle(&session_stem(&session), meta("t")); + assert!(!handle.record_partial(&TranscriptPartial::new("x"), None).unwrap()); + handle.seal().unwrap(); + assert_eq!( + entries(&docs, &session_stem(&session)) + .iter() + .filter(|entry| entry["seal"] == json!(true)) + .count(), + 1, + "sealing twice writes one seal" + ); +} + +#[test] +fn a_generation_is_opened_once() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + locator + .open_session(&session, meta("t")) + .unwrap() + .append(message("user", "a")) + .unwrap(); + let (successor, next) = locator.begin_generation(&session, meta("t")).unwrap(); + let error = locator.begin_generation(&session, meta("t")).unwrap_err(); + assert!(error.to_string().contains("reserved"), "{error}"); + next.append(message("user", "summary")).unwrap(); + let error = locator.begin_generation(&session, meta("t")).unwrap_err(); + assert!(error.to_string().contains("already exists"), "{error}"); + assert_eq!(locator.head_generation(&session), successor); + let read = next.read_session().unwrap().unwrap(); + assert_eq!(read.meta.session_id, Some(successor.session_id())); + assert_eq!(read.meta.parent_session_id, successor.parent_session_id()); +} + +#[test] +fn a_stale_reservation_is_taken_over() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + locator + .open_session(&session, meta("t")) + .unwrap() + .append(message("user", "a")) + .unwrap(); + locator.begin_generation(&session, meta("t")).unwrap(); + // The process that reserved generation 1 stopped before writing it. + let id = doc_key(&[&session_stem(&session.next_generation())]); + let aged = Arc::clone(&docs); + on_bridge(async move { + let found = aged.get(INDEX, &id).await?.unwrap(); + let mut doc = found.doc; + doc["reserved_at"] = json!(0); + aged.put(INDEX, &id, doc, Precondition::None).await + }); + let (successor, next) = locator.begin_generation(&session, meta("t")).unwrap(); + next.append(message("user", "summary")).unwrap(); + assert_eq!(locator.head_generation(&session), successor); +} + +#[test] +fn the_generation_limit_holds() { + let docs = docs(); + let mut session = SessionRef::scoped("t", "planner"); + session.generation = MAX_GENERATIONS; + let error = locator(&docs) + .begin_generation(&session, meta("t")) + .unwrap_err(); + assert!(error.to_string().contains("limit"), "{error}"); +} + +#[test] +fn lookups_find_the_newest_written_root() { + let docs = docs(); + let locator = locator(&docs); + assert!(locator.root_for_thread(" ").is_none()); + assert!( + !locator + .append_interrupted_partial(" ", None, &TranscriptPartial::new("x"), None) + .unwrap() + ); + assert!( + !locator + .append_interrupted_partial("t", None, &TranscriptPartial::new(""), None) + .unwrap() + ); + + // An opened but unwritten stem is not found. + let _unwritten = locator.open_stem("never", meta("t")).unwrap(); + assert!(locator.root_for_thread("t").is_none()); + + locator + .open_stem("old", meta("t")) + .unwrap() + .append(message("user", "old")) + .unwrap(); + locator + .open_stem("new", meta("t")) + .unwrap() + .append(message("user", "new")) + .unwrap(); + locator + .open_stem("new__child", meta("t")) + .unwrap() + .append(message("user", "child")) + .unwrap(); + let newest = locator.root_for_thread("t").unwrap(); + assert_eq!(newest.path(), Path::new("memory://test/new")); + assert_eq!( + locator.latest_for_agent("planner").unwrap().path(), + Path::new("memory://test/new"), + "sub-agent transcripts are never roots" + ); + assert!(locator.root_for_thread_scoped("t", Some("other")).is_none()); + assert!(locator.latest_for_agent("nobody").is_none()); + + assert!( + locator + .append_interrupted_partial("t", Some("planner"), &TranscriptPartial::new("p"), None) + .unwrap() + ); + assert_eq!(locator.handle("new", meta("t")).partials().unwrap().len(), 1); +} + +#[test] +fn a_thread_change_moves_the_transcript_in_the_index() { + let docs = docs(); + let locator = locator(&docs); + let history = locator.open_stem("s", meta("first")).unwrap(); + let rows = vec![message("user", "a")]; + history.append_turn(turn(&[], &rows, &meta("first"))).unwrap(); + history.append_turn(turn(&rows, &rows, &meta("second"))).unwrap(); + assert!(locator.root_for_thread("first").is_none()); + assert!(locator.root_for_thread("second").is_some()); +} + +#[test] +fn ids_are_unambiguous_and_bounded() { + assert_ne!(doc_key(&["a/b", "c"]), doc_key(&["a", "b/c"])); + let long = "x".repeat(1_000); + let hashed = doc_key(&[&long]); + assert!(hashed.starts_with("h:")); + assert!(entry_id(&long, 9).len() < tinystoragedrivers_core::MAX_ID_LEN); + assert_eq!(entry_id("s", 7), "1:s#0000000007"); + assert!(is_subagent("parent__child")); + assert!(!is_subagent("__child")); + assert!(!is_subagent("plain")); +} + +#[test] +fn a_long_stem_still_round_trips() { + let docs = docs(); + let stem = "s".repeat(600); + let history = locator(&docs).open_stem(&stem, meta("t")).unwrap(); + history.append(message("user", "a")).unwrap(); + assert_eq!(history.messages().unwrap().len(), 1); +} + +#[test] +fn an_unreadable_entry_is_an_error() { + let docs = docs(); + let locator = locator(&docs); + let history = locator.open_stem("s", meta("t")).unwrap(); + history.append(message("user", "a")).unwrap(); + let raw = Arc::clone(&docs); + on_bridge(async move { + raw.put( + ENTRIES, + &entry_id("s", 1), + json!({ "stem": "s", "seq": 1, "set": "not rows" }), + Precondition::None, + ) + .await + }); + assert!(history.messages().is_err()); + let raw = Arc::clone(&docs); + on_bridge(async move { + raw.put( + ENTRIES, + &entry_id("t", 0), + json!({ "stem": "t" }), + Precondition::None, + ) + .await + }); + let error = locator + .open_stem("t", meta("t")) + .unwrap() + .read_session() + .unwrap_err(); + assert!(error.to_string().contains("no seq"), "{error}"); +} + +#[test] +fn the_index_keeps_its_creation_time() { + let docs = docs(); + let history = locator(&docs).open_stem("s", meta("t")).unwrap(); + let rows = vec![message("user", "a")]; + history.append_turn(turn(&[], &rows, &meta("t"))).unwrap(); + let read = Arc::clone(&docs); + let first = on_bridge(async move { read.get(INDEX, &doc_key(&["s"])).await }); + history.append_turn(turn(&rows, &rows, &meta("t2"))).unwrap(); + let read = Arc::clone(&docs); + let second = on_bridge(async move { read.get(INDEX, &doc_key(&["s"])).await }); + let (first, second) = (first.unwrap(), second.unwrap()); + assert_eq!(first.doc["created_at"], second.doc["created_at"]); + assert_eq!(second.doc["thread_id"], json!("t2")); + assert!(second.version > first.version || first.version == Version(1)); +} + +#[test] +fn handles_describe_themselves() { + let docs = docs(); + let locator = locator(&docs); + assert!(format!("{locator:?}").contains("memory://test")); + assert!(format!("{:?}", locator.handle("s", meta("t"))).contains("memory://test/s")); + assert_eq!( + locator.destination_key().as_deref(), + Some("memory://test/transcripts") + ); +} From 232b3c7ac20b0c4149be375ae546eb507140b9a8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:26:08 +0300 Subject: [PATCH 26/56] test(session): cover turn state transitions Add tests for the turn state driver covering the transitions between states and the events that trigger them. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/turn_states_tests.rs | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) diff --git a/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs b/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs index e69de29bb..8dece0258 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs @@ -0,0 +1,219 @@ +use super::*; +use tinystoragedrivers_core::{MemoryStorage, Scope, StorageBackend}; + +fn store() -> (DriverTurnStates, Arc) { + let storage = MemoryStorage::new(); + let docs = Arc::clone(storage.for_scope(&Scope::local()).unwrap().documents()); + ( + DriverTurnStates::new(Arc::clone(&docs), Blocking::new().unwrap()), + docs, + ) +} + +fn started(thread: &str, request: &str, minute: u32) -> TurnState { + TurnState::started(thread, request, 8, &format!("2026-01-01T00:{minute:02}:00Z")) +} + +fn completed(thread: &str, request: &str, minute: u32) -> TurnState { + let mut turn = started(thread, request, minute); + turn.lifecycle = TurnLifecycle::Completed; + turn.updated_at = turn.started_at.clone(); + turn +} + +#[test] +fn completed_turns_are_kept_up_to_the_retention_limit() { + let (turns, _) = store(); + for minute in 0..(COMPLETED_RETENTION as u32 + 3) { + turns.put(&completed("t", &format!("r{minute:02}"), minute)).unwrap(); + } + turns.put(&started("t", "live", 59)).unwrap(); + let kept = turns.list_thread("t").unwrap(); + assert_eq!(kept.len(), COMPLETED_RETENTION + 1); + assert!( + turns.get_turn("t", "r00").unwrap().is_none(), + "the oldest completed turns go first" + ); + assert!(turns.get_turn("t", "live").unwrap().is_some(), "live turns stay"); +} + +#[test] +fn settling_prunes_too() { + let (turns, _) = store(); + for minute in 0..COMPLETED_RETENTION as u32 { + turns.put(&completed("t", &format!("r{minute:02}"), minute)).unwrap(); + } + turns.put(&started("t", "last", 58)).unwrap(); + assert!( + turns + .settle_turn("t", "last", TurnLifecycle::Completed, "2026-01-01T01:00:00Z") + .unwrap() + ); + assert_eq!(turns.list_thread("t").unwrap().len(), COMPLETED_RETENTION); +} + +#[test] +fn put_unless_completed_inserts_updates_and_then_holds() { + let (turns, _) = store(); + let mut turn = started("t", "r", 0); + assert!(turns.put_unless_completed(&turn).unwrap(), "a new turn is written"); + turn.lifecycle = TurnLifecycle::Completed; + assert!(turns.put_unless_completed(&turn).unwrap(), "a live turn is updated"); + assert!( + !turns.put_unless_completed(&started("t", "r", 1)).unwrap(), + "a completed turn is never overwritten conditionally" + ); + assert_eq!( + turns.get_turn("t", "r").unwrap().unwrap().lifecycle, + TurnLifecycle::Completed + ); +} + +#[test] +fn settling_needs_a_live_turn() { + let (turns, _) = store(); + let now = "2026-01-01T01:00:00Z"; + assert!( + !turns + .settle_turn("t", "missing", TurnLifecycle::Interrupted, now) + .unwrap() + ); + turns.put(&completed("t", "done", 0)).unwrap(); + assert!( + !turns + .settle_turn("t", "done", TurnLifecycle::Interrupted, now) + .unwrap() + ); + let mut live = started("t", "live", 1); + live.phase = Some(crate::turn_state::TurnPhase::Thinking); + turns.put(&live).unwrap(); + assert!( + turns + .settle_turn("t", "live", TurnLifecycle::Interrupted, now) + .unwrap() + ); + let settled = turns.get_turn("t", "live").unwrap().unwrap(); + assert_eq!(settled.lifecycle, TurnLifecycle::Interrupted); + assert_eq!(settled.phase, None); + assert_eq!(settled.updated_at, now); +} + +#[test] +fn deletes_report_whether_anything_went() { + let (turns, _) = store(); + assert!(!turns.delete("t").unwrap()); + assert!(!turns.delete_turn("t", "r").unwrap()); + turns.put(&started("t", "r1", 0)).unwrap(); + turns.put(&started("t", "r2", 1)).unwrap(); + turns.put(&started("u", "r1", 2)).unwrap(); + assert!(turns.delete("t").unwrap()); + assert!(turns.list_thread("t").unwrap().is_empty()); + assert_eq!(turns.list().unwrap().len(), 1); + assert_eq!(turns.clear_all().unwrap(), 1); + assert_eq!(turns.clear_all().unwrap(), 0); +} + +#[test] +fn list_answers_each_threads_newest_turn() { + let (turns, _) = store(); + turns.put(&started("t", "old", 0)).unwrap(); + turns.put(&started("t", "new", 5)).unwrap(); + turns.put(&started("u", "only", 1)).unwrap(); + let mut latest: Vec<(String, String)> = turns + .list() + .unwrap() + .into_iter() + .map(|turn| (turn.thread_id, turn.request_id)) + .collect(); + latest.sort(); + assert_eq!( + latest, + [ + ("t".to_string(), "new".to_string()), + ("u".to_string(), "only".to_string()) + ] + ); +} + +#[test] +fn ids_with_separators_do_not_collide() { + let (turns, _) = store(); + turns.put(&started("a/b", "c", 0)).unwrap(); + turns.put(&started("a", "b/c", 1)).unwrap(); + assert_eq!(turns.list().unwrap().len(), 2); +} + +#[test] +fn an_unreadable_snapshot_is_an_error() { + let (turns, docs) = store(); + turns.put(&started("t", "r", 0)).unwrap(); + let bridge = Blocking::new().unwrap(); + let raw = Arc::clone(&docs); + bridge + .run(async move { + raw.put(COLLECTION, &id("t", "bodiless"), json!({"thread_id": "t"}), Precondition::None) + .await + }) + .unwrap() + .unwrap(); + let error = turns.list_thread("t").unwrap_err(); + assert!(error.contains("no body"), "{error}"); + let raw = Arc::clone(&docs); + bridge + .run(async move { + raw.put( + COLLECTION, + &id("t", "bodiless"), + json!({"thread_id": "t", "turn": "nope"}), + Precondition::None, + ) + .await + }) + .unwrap() + .unwrap(); + let error = turns.get_turn("t", "bodiless").unwrap_err(); + assert!(error.contains("unreadable"), "{error}"); +} + +#[test] +fn contention_is_reported_after_the_last_attempt() { + let error = contended("t", "r"); + assert_eq!(error.kind(), ErrorKind::Conflict); + assert!(is_race(&error)); + assert!(!is_race(&StorageError::unavailable("down"))); +} + +#[test] +fn concurrent_settles_and_sweeps_never_lose_a_turn() { + let (turns, _) = store(); + for i in 0..8 { + turns.put(&started("t", &format!("r{i}"), i)).unwrap(); + } + let handles: Vec<_> = (0..4) + .map(|worker| { + let turns = turns.clone(); + std::thread::spawn(move || { + if worker == 0 { + turns.mark_all_interrupted("2026-01-01T02:00:00Z").unwrap(); + } else { + for i in 0..8 { + turns + .settle_turn( + "t", + &format!("r{i}"), + TurnLifecycle::Completed, + "2026-01-01T02:00:00Z", + ) + .unwrap(); + } + } + }) + }) + .collect(); + for handle in handles { + handle.join().unwrap(); + } + let all = turns.list_thread("t").unwrap(); + assert_eq!(all.len(), 8); + assert!(all.iter().all(is_terminal), "every turn ended up terminal"); +} From 464c6c3e6fe79dde85a850cbae6651cdb8b04940 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:26:32 +0300 Subject: [PATCH 27/56] chore(deps): add tinystoragedrivers-sqlite to the lockfile Record the new tinystoragedrivers-sqlite package and its dependency edges in Cargo.lock so the workspace resolves consistently. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index df7d8679d..b3522c84c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1887,6 +1887,7 @@ dependencies = [ "thiserror", "tinyagents-harness", "tinystoragedrivers-core", + "tinystoragedrivers-sqlite", "tinytools-agent", "tokio", "tracing", @@ -1996,6 +1997,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "tinystoragedrivers-sqlite" +version = "0.3.0" +dependencies = [ + "rusqlite", + "serde_json", + "tinystoragedrivers-core", + "tokio", +] + [[package]] name = "tinystr" version = "0.8.3" From bc8d0bd50532a810cd590ed4e40582620978b431 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:26:47 +0300 Subject: [PATCH 28/56] test(session): use err().unwrap() instead of unwrap_err() The transcript tests now extract expected errors with err().unwrap() rather than unwrap_err(), which keeps the assertions working when the success type does not implement Debug. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts_tests.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index ddf79eb4f..47bb7e930 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -244,10 +244,10 @@ fn a_generation_is_opened_once() { .append(message("user", "a")) .unwrap(); let (successor, next) = locator.begin_generation(&session, meta("t")).unwrap(); - let error = locator.begin_generation(&session, meta("t")).unwrap_err(); + let error = locator.begin_generation(&session, meta("t")).err().unwrap(); assert!(error.to_string().contains("reserved"), "{error}"); next.append(message("user", "summary")).unwrap(); - let error = locator.begin_generation(&session, meta("t")).unwrap_err(); + let error = locator.begin_generation(&session, meta("t")).err().unwrap(); assert!(error.to_string().contains("already exists"), "{error}"); assert_eq!(locator.head_generation(&session), successor); let read = next.read_session().unwrap().unwrap(); @@ -287,7 +287,8 @@ fn the_generation_limit_holds() { session.generation = MAX_GENERATIONS; let error = locator(&docs) .begin_generation(&session, meta("t")) - .unwrap_err(); + .err() + .unwrap(); assert!(error.to_string().contains("limit"), "{error}"); } From 21dab5e8c46ee1fc58281bbaa03b81bb97381f80 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:27:22 +0300 Subject: [PATCH 29/56] test(session): remove redundant unreadable entry assertion The test already asserts that reading history fails for an unreadable entry, so the extra block that rewrote the entry and re-checked the error message was redundant and has been dropped. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts_tests.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 47bb7e930..c5c666e86 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -396,22 +396,6 @@ fn an_unreadable_entry_is_an_error() { .await }); assert!(history.messages().is_err()); - let raw = Arc::clone(&docs); - on_bridge(async move { - raw.put( - ENTRIES, - &entry_id("t", 0), - json!({ "stem": "t" }), - Precondition::None, - ) - .await - }); - let error = locator - .open_stem("t", meta("t")) - .unwrap() - .read_session() - .unwrap_err(); - assert!(error.to_string().contains("no seq"), "{error}"); } #[test] From 81cd693e4237ff8a793599f35673df8df3d781bc Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:28:31 +0300 Subject: [PATCH 30/56] style(session): reformat driver modules with rustfmt Reformatted the storage driver modules and their tests to satisfy rustfmt, wrapping long expressions and reordering the feature-gated re-exports in lib.rs. No behaviour changed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/lib.rs | 8 +-- .../src/port/drivers/mod.rs | 6 +- .../src/port/drivers/mod_tests.rs | 38 ++++++++++-- .../src/port/drivers/refused.rs | 11 +++- .../src/port/drivers/refused_tests.rs | 6 +- .../src/port/drivers/transcripts.rs | 44 +++++++------ .../src/port/drivers/transcripts_tests.rs | 61 ++++++++++++++----- .../src/port/drivers/turn_states.rs | 9 ++- .../src/port/drivers/turn_states_tests.rs | 41 ++++++++++--- 9 files changed, 166 insertions(+), 58 deletions(-) diff --git a/crates/tinyagents-session/src/lib.rs b/crates/tinyagents-session/src/lib.rs index 8fe85817f..09114b5e8 100644 --- a/crates/tinyagents-session/src/lib.rs +++ b/crates/tinyagents-session/src/lib.rs @@ -108,14 +108,14 @@ pub use ops::{ record_message_with_reasoning, record_session_end, record_session_start, record_tool_call, search_sessions, set_fts_snippet_bytes, }; -#[cfg(feature = "storage-drivers")] -pub use port::{ - DriverSessionStores, DriverTranscriptHistory, DriverTranscriptLocator, DriverTurnStates, -}; pub use port::{ AgentStores, InMemorySessionStores, InMemoryTranscriptLocator, InMemoryTurnStates, SessionStoreProvider, TurnStates, }; +#[cfg(feature = "storage-drivers")] +pub use port::{ + DriverSessionStores, DriverTranscriptHistory, DriverTranscriptLocator, DriverTurnStates, +}; pub use retention::{ RetentionReport, apply_retention, prune_run_events_before, prune_run_telemetry_before, prune_sessions_before, prune_tool_calls_before, reindex_fts, trim_session_messages, diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index 4f2425402..457d6a156 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -228,7 +228,11 @@ impl SessionStoreProvider for DriverSessionStores { } fn destination_key(&self) -> Option { - Some(format!("{}://{:p}", self.backend.driver(), Arc::as_ptr(&self.backend))) + Some(format!( + "{}://{:p}", + self.backend.driver(), + Arc::as_ptr(&self.backend) + )) } } diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs index 60a864d9e..2e9589b49 100644 --- a/crates/tinyagents-session/src/port/drivers/mod_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/mod_tests.rs @@ -50,7 +50,10 @@ fn stores_work_outside_any_runtime() { #[test] fn a_valid_agent_id_is_its_own_scope() { - assert_eq!(DriverSessionStores::scope_for("agent-7").as_str(), "agent-7"); + assert_eq!( + DriverSessionStores::scope_for("agent-7").as_str(), + "agent-7" + ); } #[test] @@ -59,9 +62,17 @@ fn any_other_agent_id_maps_to_a_stable_hashed_scope() { assert!(spaced.as_str().starts_with("sha256:")); assert_eq!(spaced, DriverSessionStores::scope_for("has a space")); assert_ne!(spaced, DriverSessionStores::scope_for("has a space")); - assert!(DriverSessionStores::scope_for("").as_str().starts_with("sha256:")); + assert!( + DriverSessionStores::scope_for("") + .as_str() + .starts_with("sha256:") + ); let long = "a".repeat(300); - assert!(DriverSessionStores::scope_for(&long).as_str().starts_with("sha256:")); + assert!( + DriverSessionStores::scope_for(&long) + .as_str() + .starts_with("sha256:") + ); } #[test] @@ -129,7 +140,13 @@ fn recover_on_open_interrupts_turns_an_earlier_process_left() { let plain = DriverSessionStores::new(Arc::clone(&backend)).unwrap(); assert_eq!( - plain.for_agent("a").turn_states.get("t").unwrap().unwrap().lifecycle, + plain + .for_agent("a") + .turn_states + .get("t") + .unwrap() + .unwrap() + .lifecycle, TurnLifecycle::Started, "without the option nothing is touched" ); @@ -197,7 +214,13 @@ async fn an_unbindable_scope_fails_closed() { assert!(error.contains("database is down"), "{error}"); assert!(stores.turn_states.list().is_err()); assert!(stores.kv.get("ns", "k").await.is_err()); - assert!(stores.journal.append("s", serde_json::json!(1)).await.is_err()); + assert!( + stores + .journal + .append("s", serde_json::json!(1)) + .await + .is_err() + ); assert!(stores.journal.len("s").await.is_err()); let session = crate::transcript::SessionRef::scoped("t", "a"); @@ -217,6 +240,9 @@ async fn an_unbindable_scope_fails_closed() { .is_err() ); - assert!(provider.agents.lock().unwrap().is_empty(), "refusals are not cached"); + assert!( + provider.agents.lock().unwrap().is_empty(), + "refusals are not cached" + ); provider.recover().unwrap(); } diff --git a/crates/tinyagents-session/src/port/drivers/refused.rs b/crates/tinyagents-session/src/port/drivers/refused.rs index f6bfb3aa9..7e3968497 100644 --- a/crates/tinyagents-session/src/port/drivers/refused.rs +++ b/crates/tinyagents-session/src/port/drivers/refused.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use serde_json::Value; use tinystoragedrivers_core::{ - Blocking, Capabilities, CollectionSpec, DocumentStore, ErrorKind, Filter, Page, Precondition, Query, - Result, Sort, StorageError, StreamEntry, StreamStore, Version, Versioned, async_trait, + Blocking, Capabilities, CollectionSpec, DocumentStore, ErrorKind, Filter, Page, Precondition, + Query, Result, Sort, StorageError, StreamEntry, StreamStore, Version, Versioned, async_trait, }; use super::super::AgentStores; @@ -111,7 +111,12 @@ impl StreamStore for Refused { Err(self.error()) } - async fn read_window(&self, _stream: &str, _from: u64, _limit: usize) -> Result> { + async fn read_window( + &self, + _stream: &str, + _from: u64, + _limit: usize, + ) -> Result> { Err(self.error()) } diff --git a/crates/tinyagents-session/src/port/drivers/refused_tests.rs b/crates/tinyagents-session/src/port/drivers/refused_tests.rs index 119cfeee7..acef2fd7f 100644 --- a/crates/tinyagents-session/src/port/drivers/refused_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/refused_tests.rs @@ -16,7 +16,11 @@ async fn every_document_call_reports_the_refusal() { }; assert_eq!(docs.capabilities(), Capabilities::default()); assert_eq!( - kind(docs.ensure_collection(&CollectionSpec::new("c")).await.unwrap_err()), + kind( + docs.ensure_collection(&CollectionSpec::new("c")) + .await + .unwrap_err() + ), ErrorKind::Unavailable ); assert!(docs.get("c", "i").await.is_err()); diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index ff6ea71c6..872ba55d8 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -199,7 +199,9 @@ impl Replay { } fn normalized(rows: Vec) -> Vec { - rows.into_iter().map(TranscriptMessage::normalized).collect() + rows.into_iter() + .map(TranscriptMessage::normalized) + .collect() } /// Declares both collections, once per locator. @@ -219,7 +221,10 @@ impl Declared { docs.ensure_collection( &CollectionSpec::new(INDEX) .index(IndexSpec::new("by_thread", ["thread_id", "created_at"])) - .index(IndexSpec::new("by_agent_name", ["agent_name", "created_at"])) + .index(IndexSpec::new( + "by_agent_name", + ["agent_name", "created_at"], + )) .index(IndexSpec::new("by_agent_id", ["agent_id", "created_at"])), ) .await @@ -292,10 +297,7 @@ impl DriverTranscriptHistory { B: Fn(&Replay, &TranscriptMeta) -> anyhow::Result> + Send + 'static, { let inner = Arc::clone(&self.inner); - let outcome = run_on(&self.bridge, async move { - Ok(inner.commit(build).await) - })?; - outcome + run_on(&self.bridge, async move { Ok(inner.commit(build).await) })? } /// Seals this generation; later writes to it are refused. @@ -343,9 +345,16 @@ impl HistoryInner { .sort(Sort::asc("seq")) .limit(500); for stored in self.docs.query_all(ENTRIES, &query).await? { - let seq = stored.doc.get("seq").and_then(Value::as_u64).ok_or_else(|| { - StorageError::serialization(format!("transcript entry {} has no seq", stored.id)) - })?; + let seq = stored + .doc + .get("seq") + .and_then(Value::as_u64) + .ok_or_else(|| { + StorageError::serialization(format!( + "transcript entry {} has no seq", + stored.id + )) + })?; let entry: Entry = serde_json::from_value(stored.doc).map_err(|error| { StorageError::serialization(format!( "transcript entry {} is unreadable: {error}", @@ -375,7 +384,12 @@ impl HistoryInner { } match self .docs - .put(ENTRIES, &entry_id(&self.stem, seq), doc, Precondition::Absent) + .put( + ENTRIES, + &entry_id(&self.stem, seq), + doc, + Precondition::Absent, + ) .await { Ok(_) => { @@ -669,10 +683,7 @@ impl DriverTranscriptLocator { /// Reserves `stem` as a new generation: succeeds when nothing is written /// there and no live reservation holds it. - async fn reserve( - docs: &Arc, - stem: &str, - ) -> anyhow::Result<()> { + async fn reserve(docs: &Arc, stem: &str) -> anyhow::Result<()> { let written = docs .count( ENTRIES, @@ -860,10 +871,7 @@ impl TranscriptLocator for DriverTranscriptLocator { }; reserved?; let predecessor_stem = session_stem(session); - if let Err(error) = self - .handle(&predecessor_stem, seed.clone()) - .seal() - { + if let Err(error) = self.handle(&predecessor_stem, seed.clone()).seal() { let docs = Arc::clone(&self.docs); let id = doc_key(&[&stem]); if let Err(release) = run_on(&self.bridge, async move { diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index c5c666e86..6292ab96e 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -81,7 +81,9 @@ fn an_ordinary_turn_stores_only_its_new_rows() { history.append_turn(turn(&[], &first, &meta("t"))).unwrap(); let mut second = first.clone(); second.extend([message("user", "c"), message("assistant", "d")]); - history.append_turn(turn(&first, &second, &meta("t"))).unwrap(); + history + .append_turn(turn(&first, &second, &meta("t"))) + .unwrap(); let compacted = vec![message("user", "summary")]; history .append_turn(turn(&second, &compacted, &meta("t"))) @@ -159,7 +161,11 @@ fn tools_and_meta_follow_the_latest_turn() { later.turn_count = 2; history.append_turn(turn(&rows, &rows, &later)).unwrap(); let read = history.read_session().unwrap().unwrap(); - assert_eq!(read.tools, Some(tools), "a turn without tools keeps the last"); + assert_eq!( + read.tools, + Some(tools), + "a turn without tools keeps the last" + ); assert_eq!(read.meta.turn_count, 2); } @@ -168,7 +174,11 @@ fn partials_stay_out_of_the_replay_and_a_clear_drops_them() { let docs = docs(); let locator = locator(&docs); let handle = locator.handle("s", meta("t")); - assert!(!handle.record_partial(&TranscriptPartial::new(""), None).unwrap()); + assert!( + !handle + .record_partial(&TranscriptPartial::new(""), None) + .unwrap() + ); let rows = vec![message("user", "a")]; let mut partial = TranscriptPartial::new("half"); partial.reasoning_content = Some("thinking".into()); @@ -176,7 +186,11 @@ fn partials_stay_out_of_the_replay_and_a_clear_drops_them() { handle .append_turn_with_partial(turn(&[], &rows, &meta("t")), Some(&partial)) .unwrap(); - assert!(handle.record_partial(&TranscriptPartial::new("more"), Some("r2")).unwrap()); + assert!( + handle + .record_partial(&TranscriptPartial::new("more"), Some("r2")) + .unwrap() + ); let partials = handle.partials().unwrap(); assert_eq!(partials.len(), 2); assert_eq!(partials[0], (partial, Some("req".to_string()))); @@ -195,10 +209,18 @@ fn partials_stay_out_of_the_replay_and_a_clear_drops_them() { fn a_partial_alone_makes_the_transcript_exist() { let docs = docs(); let handle = locator(&docs).handle("s", meta("t")); - assert!(handle.record_partial(&TranscriptPartial::new("half"), None).unwrap()); + assert!( + handle + .record_partial(&TranscriptPartial::new("half"), None) + .unwrap() + ); let read = handle.read_session().unwrap().unwrap(); assert!(read.messages.is_empty()); - assert_eq!(read.meta.thread_id.as_deref(), Some("t"), "the seed is the meta"); + assert_eq!( + read.meta.thread_id.as_deref(), + Some("t"), + "the seed is the meta" + ); } #[test] @@ -214,14 +236,16 @@ fn a_sealed_generation_refuses_writes() { history.append(message("user", "late")).unwrap_err(), history.replace(&[]).unwrap_err(), history.clear().unwrap_err(), - history - .append_turn(turn(&[], &[], &meta("t"))) - .unwrap_err(), + history.append_turn(turn(&[], &[], &meta("t"))).unwrap_err(), ] { assert!(error.to_string().contains("sealed"), "{error}"); } let handle = locator.handle(&session_stem(&session), meta("t")); - assert!(!handle.record_partial(&TranscriptPartial::new("x"), None).unwrap()); + assert!( + !handle + .record_partial(&TranscriptPartial::new("x"), None) + .unwrap() + ); handle.seal().unwrap(); assert_eq!( entries(&docs, &session_stem(&session)) @@ -342,7 +366,10 @@ fn lookups_find_the_newest_written_root() { .append_interrupted_partial("t", Some("planner"), &TranscriptPartial::new("p"), None) .unwrap() ); - assert_eq!(locator.handle("new", meta("t")).partials().unwrap().len(), 1); + assert_eq!( + locator.handle("new", meta("t")).partials().unwrap().len(), + 1 + ); } #[test] @@ -351,8 +378,12 @@ fn a_thread_change_moves_the_transcript_in_the_index() { let locator = locator(&docs); let history = locator.open_stem("s", meta("first")).unwrap(); let rows = vec![message("user", "a")]; - history.append_turn(turn(&[], &rows, &meta("first"))).unwrap(); - history.append_turn(turn(&rows, &rows, &meta("second"))).unwrap(); + history + .append_turn(turn(&[], &rows, &meta("first"))) + .unwrap(); + history + .append_turn(turn(&rows, &rows, &meta("second"))) + .unwrap(); assert!(locator.root_for_thread("first").is_none()); assert!(locator.root_for_thread("second").is_some()); } @@ -406,7 +437,9 @@ fn the_index_keeps_its_creation_time() { history.append_turn(turn(&[], &rows, &meta("t"))).unwrap(); let read = Arc::clone(&docs); let first = on_bridge(async move { read.get(INDEX, &doc_key(&["s"])).await }); - history.append_turn(turn(&rows, &rows, &meta("t2"))).unwrap(); + history + .append_turn(turn(&rows, &rows, &meta("t2"))) + .unwrap(); let read = Arc::clone(&docs); let second = on_bridge(async move { read.get(INDEX, &doc_key(&["s"])).await }); let (first, second) = (first.unwrap(), second.unwrap()); diff --git a/crates/tinyagents-session/src/port/drivers/turn_states.rs b/crates/tinyagents-session/src/port/drivers/turn_states.rs index 3a14b59e5..fa2f492a0 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states.rs @@ -25,7 +25,9 @@ use tinystoragedrivers_core::{ use tokio::sync::OnceCell; use super::super::TurnStates; -use super::super::memory::{COMPLETED_RETENTION, completed_newest_first, is_terminal, newest_first}; +use super::super::memory::{ + COMPLETED_RETENTION, completed_newest_first, is_terminal, newest_first, +}; use super::transcripts::doc_key; use crate::turn_state::{TurnLifecycle, TurnState}; @@ -307,7 +309,10 @@ impl TurnStates for DriverTurnStates { let key = id(thread_id, request_id); self.run(|inner| async move { inner.declared().await?; - inner.docs.delete(COLLECTION, &key, Precondition::None).await + inner + .docs + .delete(COLLECTION, &key, Precondition::None) + .await }) } diff --git a/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs b/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs index 8dece0258..f271acadb 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states_tests.rs @@ -11,7 +11,7 @@ fn store() -> (DriverTurnStates, Arc) { } fn started(thread: &str, request: &str, minute: u32) -> TurnState { - TurnState::started(thread, request, 8, &format!("2026-01-01T00:{minute:02}:00Z")) + TurnState::started(thread, request, 8, format!("2026-01-01T00:{minute:02}:00Z")) } fn completed(thread: &str, request: &str, minute: u32) -> TurnState { @@ -25,7 +25,9 @@ fn completed(thread: &str, request: &str, minute: u32) -> TurnState { fn completed_turns_are_kept_up_to_the_retention_limit() { let (turns, _) = store(); for minute in 0..(COMPLETED_RETENTION as u32 + 3) { - turns.put(&completed("t", &format!("r{minute:02}"), minute)).unwrap(); + turns + .put(&completed("t", &format!("r{minute:02}"), minute)) + .unwrap(); } turns.put(&started("t", "live", 59)).unwrap(); let kept = turns.list_thread("t").unwrap(); @@ -34,19 +36,29 @@ fn completed_turns_are_kept_up_to_the_retention_limit() { turns.get_turn("t", "r00").unwrap().is_none(), "the oldest completed turns go first" ); - assert!(turns.get_turn("t", "live").unwrap().is_some(), "live turns stay"); + assert!( + turns.get_turn("t", "live").unwrap().is_some(), + "live turns stay" + ); } #[test] fn settling_prunes_too() { let (turns, _) = store(); for minute in 0..COMPLETED_RETENTION as u32 { - turns.put(&completed("t", &format!("r{minute:02}"), minute)).unwrap(); + turns + .put(&completed("t", &format!("r{minute:02}"), minute)) + .unwrap(); } turns.put(&started("t", "last", 58)).unwrap(); assert!( turns - .settle_turn("t", "last", TurnLifecycle::Completed, "2026-01-01T01:00:00Z") + .settle_turn( + "t", + "last", + TurnLifecycle::Completed, + "2026-01-01T01:00:00Z" + ) .unwrap() ); assert_eq!(turns.list_thread("t").unwrap().len(), COMPLETED_RETENTION); @@ -56,9 +68,15 @@ fn settling_prunes_too() { fn put_unless_completed_inserts_updates_and_then_holds() { let (turns, _) = store(); let mut turn = started("t", "r", 0); - assert!(turns.put_unless_completed(&turn).unwrap(), "a new turn is written"); + assert!( + turns.put_unless_completed(&turn).unwrap(), + "a new turn is written" + ); turn.lifecycle = TurnLifecycle::Completed; - assert!(turns.put_unless_completed(&turn).unwrap(), "a live turn is updated"); + assert!( + turns.put_unless_completed(&turn).unwrap(), + "a live turn is updated" + ); assert!( !turns.put_unless_completed(&started("t", "r", 1)).unwrap(), "a completed turn is never overwritten conditionally" @@ -151,8 +169,13 @@ fn an_unreadable_snapshot_is_an_error() { let raw = Arc::clone(&docs); bridge .run(async move { - raw.put(COLLECTION, &id("t", "bodiless"), json!({"thread_id": "t"}), Precondition::None) - .await + raw.put( + COLLECTION, + &id("t", "bodiless"), + json!({"thread_id": "t"}), + Precondition::None, + ) + .await }) .unwrap() .unwrap(); From 01fc9c001d68dff02470bd461e0ac10e7432b245 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:29:14 +0300 Subject: [PATCH 31/56] docs(session): document the driver-backed store port The session module README now points at the store-port document, and that document describes `DriverSessionStores`: how each agent maps to a storage scope, the transcript log and generation reservation rules, compare-and-swap turn states, the blocking sync bridge, recovery limits and fail-closed behaviour. Conformance notes were updated to record that the driver provider runs the shared and isolation suites over the memory and SQLite drivers. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/README.md | 4 ++ docs/modules/session/store-port.md | 58 ++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/crates/tinyagents-session/src/README.md b/crates/tinyagents-session/src/README.md index f76dad831..846c414e1 100644 --- a/crates/tinyagents-session/src/README.md +++ b/crates/tinyagents-session/src/README.md @@ -68,6 +68,10 @@ reachable under `session::`, `session::run_ledger::`, and - **Chat threads** — `ConversationStore` and its wire types (re-exported at the root), plus the free functions and channel subscriber under `session::threads` — see its own [README](./threads/README.md) +- **Store port** — `SessionStoreProvider`, `AgentStores`, `TurnStates`, + `InMemorySessionStores`, and with feature `storage-drivers` + `DriverSessionStores` over a `tinystoragedrivers` backend — see + [`docs/modules/session/store-port.md`](../../../docs/modules/session/store-port.md) - **Connections** — `with_connection` (autocommit) and `with_transaction` (`BEGIN IMMEDIATE`) - **Testkit** — `testkit::conformance::run_ledger_conformance` and diff --git a/docs/modules/session/store-port.md b/docs/modules/session/store-port.md index e92d83e4c..6afc6552e 100644 --- a/docs/modules/session/store-port.md +++ b/docs/modules/session/store-port.md @@ -41,8 +41,8 @@ not claim isolation. `TranscriptLocator`/`TranscriptHistory` and `TurnStates` are synchronous: the turn path that commits a transcript is a chain of sync methods. An -implementation over an async client bridges at its own boundary (for tokio, -`block_in_place` on a multi-threaded runtime). `Store` and `AppendStore` are +implementation over an async client bridges at its own boundary (`DriverSessionStores` uses a dedicated runtime thread; +`block_in_place` also works, on a multi-threaded tokio runtime only). `Store` and `AppendStore` are the harness's async traits; `Arc` and `Arc` implement them, so code generic over a store accepts injected handles, and `FileStatusStore::over` keeps run status in any `Store`. @@ -59,6 +59,53 @@ implement them, so code generic over a store accepts injected handles, and `openhuman_rpc::session_store`). - `TranscriptLocator::append_interrupted_partial` carries the display-only partial of an interrupted turn without a file path. +- `DriverSessionStores` (feature `storage-drivers`): every agent's stores in + one `tinystoragedrivers` backend, described below. + +## `DriverSessionStores` + +With the `storage-drivers` feature, a host that has opened a +`tinystoragedrivers` backend (SQLite on a desktop, MongoDB in the cloud, +memory in tests) gets a complete provider from it: + +```rust +let backend: Arc = Arc::new(SqliteStorage::open(path)?); +let provider = DriverSessionStores::new(backend)?.recover_on_open(true); +``` + +Each agent id becomes a storage `Scope` (an id that is not a valid scope +maps to `sha256:` of itself), and the driver enforces scopes, so the +provider claims isolation and passes `session_store_isolation_conformance`. + +| Store | Backend shape | +| --- | --- | +| transcripts | `session_transcripts`: one index document per stem (thread, agent, sub-agent flag, creation time); `session_transcript_entries`: an append-only log per stem | +| turn states | `session_turn_states`: one document per `(thread, request)` | +| key-value | `session_kv` through the harness `DriverStore` | +| journal | streams `session_journal/` through the harness `DriverAppendStore` | + +- **Transcript log.** Each write is one entry, numbered from 0 and claimed + with an insert-only write. An ordinary turn stores only its new rows + (`extend`); a first write or a compaction stores the whole set (`set`). Two + writers on one transcript (two processes on one database) serialize: the + loser re-reads and decides again, which is also how a write racing a seal is + refused. +- **Generations.** `begin_generation` reserves the successor in the index + before sealing the predecessor, so two compactions cannot both open it. A + reservation left unwritten for 30 seconds (its process stopped mid-way) may + be taken over, so a sealed head never strands the conversation. +- **Turn states.** Conditional writes, settling and the interruption sweep + are compare-and-swap on the document version. +- **Sync seams.** Transcript and turn-state calls run on a + `tinystoragedrivers` `Blocking` bridge (one dedicated runtime thread), so + they work from any caller, inside a runtime or not. +- **Recovery.** A backend cannot list its scopes, so `recover` covers the + agents this provider has opened. `recover_on_open(true)` also interrupts an + agent's in-flight turns the first time it is opened. Use it only when a + single process owns the database (the desktop app). +- **Failing closed.** If the backend cannot bind an agent's scope, `for_agent` + returns stores that refuse every call with that error and does not cache + them. `try_for_agent` returns the error itself. ## Conformance @@ -66,8 +113,9 @@ implement them, so code generic over a store accepts injected handles, and whole provider: transcripts (sessions, thread and agent lookups, compaction generations, partials kept out of the replay), turn states (conditional writes, settling, interrupted-marking) and the key-value and journal stores. -`session_store_conformance` runs against `InMemorySessionStores` and the file -building blocks. It checks the common session-store behavior, not provider +`session_store_conformance` runs against `InMemorySessionStores`, the file +building blocks and `DriverSessionStores` over the memory and SQLite drivers. It checks the common session-store behavior, not provider isolation. `session_store_isolation_conformance` is a separate check that two agents cannot see each other's data; it runs only against -`InMemorySessionStores` and hosts whose providers claim isolation. +`InMemorySessionStores`, `DriverSessionStores` and hosts whose providers claim +isolation. From b372ba107f3a894ccc7bb4b16a0c053d3efb4a04 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:29:24 +0300 Subject: [PATCH 32/56] docs(agents): document storage-drivers feature Update the Cargo feature list in AGENTS.md to include the new `storage-drivers` feature on the harness and graph crates, and note that session now exposes it while registry only exposes `tracing`. The feature routes each store through `tinystoragedrivers` ports. Auto-committed-on: dragonfly Co-authored-by: Medulla --- AGENTS.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f22a6fb6b..76418a921 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,8 +21,10 @@ dedicated `types.rs` file and keep module-local unit tests in a sibling smallest useful API. Cargo features are package-local. `tinyagents-harness` exposes `sqlite`, -`tools`, `multimodal`, and `tracing`; `tinyagents-graph` exposes `sqlite` and -`tracing`; registry and session expose `tracing`. Tracing instrumentation is +`tools`, `multimodal`, `storage-drivers`, and `tracing`; `tinyagents-graph` +exposes `sqlite`, `storage-drivers`, and `tracing`; session exposes +`storage-drivers` and `tracing`; registry exposes `tracing`. The +`storage-drivers` features put each store on `tinystoragedrivers` ports. Tracing instrumentation is compiled out by default. Integration tests are in `crates/tinyagents-integration-tests/tests/`, covering serialization, graph routing, From ccb4c1e1cbd2d10286ef296fd49404f138f1795c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:31:20 +0300 Subject: [PATCH 33/56] feat(tinyagents-graph): add checkpoint driver abstraction Introduce a checkpoint driver layer so graph state can be persisted and restored through a pluggable backend instead of being tied to a single storage implementation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-graph/Cargo.toml | 4 +- .../src/checkpoint/drivers.rs | 72 +++++++++++++++---- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/crates/tinyagents-graph/Cargo.toml b/crates/tinyagents-graph/Cargo.toml index e8e96d240..894a68ffe 100644 --- a/crates/tinyagents-graph/Cargo.toml +++ b/crates/tinyagents-graph/Cargo.toml @@ -24,13 +24,15 @@ tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-mult tracing = { workspace = true } # Storage ports (no database client) for `DriverCheckpointer`. tinystoragedrivers-core = { workspace = true, optional = true } +# Collision-resistant ids for `DriverCheckpointer` keys too long to store. +sha2 = { version = "0.11", optional = true } [features] default = [] sqlite = ["dep:rusqlite"] # `DriverCheckpointer`: checkpoints, pending writes and leases on any # tinystoragedrivers backend the host opened. -storage-drivers = ["dep:tinystoragedrivers-core"] +storage-drivers = ["dep:tinystoragedrivers-core", "dep:sha2"] # Tracing instrumentation is now always compiled in (via the `tracing` crate # dependency above). This feature is retained as a no-op so downstream # feature forwards keep compiling. diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs index b5ba9e4f6..d6d154f69 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -11,7 +11,8 @@ //! Three collections, named from a prefix (default `graph`): //! //! - `_checkpoints`: one document per stored checkpoint, -//! `{thread, seq, checkpoint_id, record}`. `seq` is a per-thread insertion +//! `{thread, namespace, seq, checkpoint_id, record}` (`namespace` is the +//! encoded subgraph namespace, so scoped reads filter on it). `seq` is a per-thread insertion //! counter, so listing is a query sorted on it and duplicate checkpoint ids //! resolve to the latest write, exactly like the append-only backends. //! - `_threads`: one counter document per thread, advanced with a @@ -49,28 +50,41 @@ fn map_error(error: StorageError) -> TinyAgentsError { TinyAgentsError::Checkpoint(format!("storage driver: {error}")) } -/// A stable 64-bit FNV-1a hash, for ids built from unbounded keys. -fn fnv1a(bytes: &[u8]) -> u64 { - bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { - (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) - }) -} +/// Documents read per driver call when following a query to its end. +/// [`DocumentStoreExt::query_all`] follows the cursors, so this bounds one +/// page, not the result. +const PAGE: usize = 500; + +/// Longest id stored as is; longer ones are hashed. +const MAX_KEY_LEN: usize = 400; /// A document id for `parts`: length-prefixed so no two tuples collide, and -/// hashed when it would exceed the driver's id limit. +/// replaced by its SHA-256 when it would exceed the driver's id limit. fn key(parts: &[&str]) -> String { let joined: String = parts .iter() .map(|part| format!("{}:{part}", part.len())) .collect::>() .join("/"); - if joined.len() <= 400 { + if joined.len() <= MAX_KEY_LEN { joined } else { - format!("h:{:016x}:{}", fnv1a(joined.as_bytes()), joined.len()) + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(joined.as_bytes()); + let hex: String = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + format!("h:{hex}") } } +/// `namespace` as one string, injectively: each component is +/// length-prefixed, so `["a", "b"]` and `["a/b"]` never meet. +fn namespace_key(namespace: &[String]) -> String { + namespace + .iter() + .map(|part| format!("{}:{part};", part.len())) + .collect() +} + /// A [`Checkpointer`] that stores everything in a driver [`DocumentStore`]. pub struct DriverCheckpointer { docs: Arc, @@ -130,7 +144,11 @@ impl DriverCheckpointer { let specs = [ CollectionSpec::new(&self.checkpoints) .index(IndexSpec::new("by_thread_seq", ["thread", "seq"])) - .index(IndexSpec::new("by_thread_id", ["thread", "checkpoint_id"])), + .index(IndexSpec::new("by_thread_id", ["thread", "checkpoint_id"])) + .index(IndexSpec::new( + "by_thread_namespace", + ["thread", "namespace", "seq"], + )), CollectionSpec::new(&self.threads), CollectionSpec::new(&self.writes) .index(IndexSpec::new("by_thread", ["thread"])), @@ -174,7 +192,7 @@ impl DriverCheckpointer { self.declared().await?; let query = Query::filter(Filter::eq("thread", thread)) .sort(Sort::asc("seq")) - .limit(500); + .limit(PAGE); self.docs .query_all(&self.checkpoints, &query) .await @@ -182,8 +200,7 @@ impl DriverCheckpointer { } fn writes_id(thread: &str, namespace: &[String], checkpoint_id: &str) -> String { - let namespace = namespace.join("\u{1f}"); - key(&[thread, &namespace, checkpoint_id]) + key(&[thread, &namespace_key(namespace), checkpoint_id]) } async fn drop_writes(&self, filter: Filter) -> Result<()> { @@ -221,6 +238,7 @@ where let seq = self.next_seq(&checkpoint.thread_id).await?; let doc = json!({ "thread": checkpoint.thread_id, + "namespace": namespace_key(&checkpoint.namespace), "seq": seq, "checkpoint_id": checkpoint.checkpoint_id, "record": serde_json::to_value(&checkpoint)?, @@ -252,6 +270,30 @@ where page.items.into_iter().next().map(Self::decode).transpose() } + /// One indexed query on the stored namespace, so a parent run and a + /// subgraph sharing a thread never load each other's checkpoints, even + /// when they reuse a checkpoint id. + async fn get_scoped( + &self, + thread_id: &str, + checkpoint_id: Option<&str>, + namespace: &[String], + ) -> Result>> { + self.declared().await?; + let mut filter = + Filter::eq("thread", thread_id).and(Filter::eq("namespace", namespace_key(namespace))); + if let Some(id) = checkpoint_id { + filter = filter.and(Filter::eq("checkpoint_id", id)); + } + let query = Query::filter(filter).sort(Sort::desc("seq")).limit(1); + let page = self + .docs + .query(&self.checkpoints, &query) + .await + .map_err(map_error)?; + page.items.into_iter().next().map(Self::decode).transpose() + } + async fn list(&self, thread_id: &str) -> Result> { Ok(self .get_thread(thread_id) @@ -273,7 +315,7 @@ where self.declared().await?; let counters = self .docs - .query_all(&self.threads, &Query::all().limit(500)) + .query_all(&self.threads, &Query::all().limit(PAGE)) .await .map_err(map_error)?; let mut threads = Vec::new(); From 0310401d0453010ad8a538f6431fb2d7bb947df4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:32:15 +0300 Subject: [PATCH 34/56] test(checkpoint): cover namespace encoding and scoped read isolation Add tests asserting that namespace keys encode injectively, that scoped reads stay within their namespace when checkpoint ids repeat, and that pending writes do not merge across lookalike namespaces. Also tighten the long-key hash assertion to the exact expected length and verify the hash covers the length-prefixed tuple. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 1 + .../src/checkpoint/drivers_tests.rs | 64 ++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b3522c84c..a9cf3ccc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1717,6 +1717,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2", "tempfile", "tinyagents-harness", "tinyinference-llm", diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs index cdf1881c6..df1df4f05 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -111,8 +111,70 @@ fn long_keys_are_hashed_and_tuples_never_collide() { assert_ne!(key(&["a/b", "c"]), key(&["a", "b/c"])); let long = "t".repeat(500); let hashed = key(&[&long]); - assert!(hashed.starts_with("h:") && hashed.len() < 64, "{hashed}"); + assert!(hashed.starts_with("h:") && hashed.len() == 66, "{hashed}"); assert_ne!(hashed, key(&[&"u".repeat(500)])); + assert_ne!( + key(&[&"t".repeat(500)]), + key(&[&"t".repeat(499), "t"]), + "the hash covers the length-prefixed tuple" + ); +} + +#[test] +fn namespaces_encode_injectively() { + let ns = |parts: &[&str]| parts.iter().map(|p| (*p).to_string()).collect::>(); + assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a\u{1f}b"]))); + assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a/b"]))); + assert_ne!(namespace_key(&ns(&["ab"])), namespace_key(&ns(&["a", "b"]))); + assert_eq!(namespace_key(&[]), ""); +} + +#[tokio::test] +async fn scoped_reads_stay_in_their_namespace_when_ids_repeat() { + let saver = checkpointer(); + let child = vec!["sub".to_string()]; + saver.put(sample("t", "same", None, 1)).await.unwrap(); + saver + .put(sample("t", "same", None, 2).with_namespace(child.clone())) + .await + .unwrap(); + let root = saver.get_scoped("t", Some("same"), &[]).await.unwrap().unwrap(); + assert_eq!(root.state, 1, "the root never loads the subgraph's checkpoint"); + let nested = saver + .get_scoped("t", Some("same"), &child) + .await + .unwrap() + .unwrap(); + assert_eq!(nested.state, 2); + assert_eq!(saver.get_scoped("t", None, &[]).await.unwrap().unwrap().state, 1); + assert!( + saver + .get_scoped("t", None, &["other".to_string()]) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn pending_writes_never_merge_across_lookalike_namespaces() { + let saver = checkpointer(); + let config = |namespace: Vec| CheckpointConfig { + thread_id: "t".to_string(), + checkpoint_id: Some("c".to_string()), + namespace, + }; + let split = config(vec!["a".to_string(), "b".to_string()]); + let joined = config(vec!["a\u{1f}b".to_string()]); + saver + .put_writes( + &split, + &[PendingWrite::data("n", "task", 0, "out", json!("split"))], + ) + .await + .unwrap(); + assert!(saver.get_writes(&joined).await.unwrap().is_empty()); + assert_eq!(saver.get_writes(&split).await.unwrap().len(), 1); } #[test] From 3adbef265486ec3fddc39c50d2252e2cf06d3f04 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:31:20 +0300 Subject: [PATCH 35/56] feat(tinyagents-graph): add checkpoint driver abstraction Introduce a checkpoint driver layer so graph state can be persisted and restored through a pluggable backend instead of being tied to a single storage implementation. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-graph/Cargo.toml | 4 +- .../src/checkpoint/drivers.rs | 72 +++++++++++++++---- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/crates/tinyagents-graph/Cargo.toml b/crates/tinyagents-graph/Cargo.toml index e8e96d240..894a68ffe 100644 --- a/crates/tinyagents-graph/Cargo.toml +++ b/crates/tinyagents-graph/Cargo.toml @@ -24,13 +24,15 @@ tokio = { workspace = true, features = ["sync", "time", "macros", "rt", "rt-mult tracing = { workspace = true } # Storage ports (no database client) for `DriverCheckpointer`. tinystoragedrivers-core = { workspace = true, optional = true } +# Collision-resistant ids for `DriverCheckpointer` keys too long to store. +sha2 = { version = "0.11", optional = true } [features] default = [] sqlite = ["dep:rusqlite"] # `DriverCheckpointer`: checkpoints, pending writes and leases on any # tinystoragedrivers backend the host opened. -storage-drivers = ["dep:tinystoragedrivers-core"] +storage-drivers = ["dep:tinystoragedrivers-core", "dep:sha2"] # Tracing instrumentation is now always compiled in (via the `tracing` crate # dependency above). This feature is retained as a no-op so downstream # feature forwards keep compiling. diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs index b5ba9e4f6..d6d154f69 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -11,7 +11,8 @@ //! Three collections, named from a prefix (default `graph`): //! //! - `_checkpoints`: one document per stored checkpoint, -//! `{thread, seq, checkpoint_id, record}`. `seq` is a per-thread insertion +//! `{thread, namespace, seq, checkpoint_id, record}` (`namespace` is the +//! encoded subgraph namespace, so scoped reads filter on it). `seq` is a per-thread insertion //! counter, so listing is a query sorted on it and duplicate checkpoint ids //! resolve to the latest write, exactly like the append-only backends. //! - `_threads`: one counter document per thread, advanced with a @@ -49,28 +50,41 @@ fn map_error(error: StorageError) -> TinyAgentsError { TinyAgentsError::Checkpoint(format!("storage driver: {error}")) } -/// A stable 64-bit FNV-1a hash, for ids built from unbounded keys. -fn fnv1a(bytes: &[u8]) -> u64 { - bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { - (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) - }) -} +/// Documents read per driver call when following a query to its end. +/// [`DocumentStoreExt::query_all`] follows the cursors, so this bounds one +/// page, not the result. +const PAGE: usize = 500; + +/// Longest id stored as is; longer ones are hashed. +const MAX_KEY_LEN: usize = 400; /// A document id for `parts`: length-prefixed so no two tuples collide, and -/// hashed when it would exceed the driver's id limit. +/// replaced by its SHA-256 when it would exceed the driver's id limit. fn key(parts: &[&str]) -> String { let joined: String = parts .iter() .map(|part| format!("{}:{part}", part.len())) .collect::>() .join("/"); - if joined.len() <= 400 { + if joined.len() <= MAX_KEY_LEN { joined } else { - format!("h:{:016x}:{}", fnv1a(joined.as_bytes()), joined.len()) + use sha2::{Digest, Sha256}; + let digest = Sha256::digest(joined.as_bytes()); + let hex: String = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + format!("h:{hex}") } } +/// `namespace` as one string, injectively: each component is +/// length-prefixed, so `["a", "b"]` and `["a/b"]` never meet. +fn namespace_key(namespace: &[String]) -> String { + namespace + .iter() + .map(|part| format!("{}:{part};", part.len())) + .collect() +} + /// A [`Checkpointer`] that stores everything in a driver [`DocumentStore`]. pub struct DriverCheckpointer { docs: Arc, @@ -130,7 +144,11 @@ impl DriverCheckpointer { let specs = [ CollectionSpec::new(&self.checkpoints) .index(IndexSpec::new("by_thread_seq", ["thread", "seq"])) - .index(IndexSpec::new("by_thread_id", ["thread", "checkpoint_id"])), + .index(IndexSpec::new("by_thread_id", ["thread", "checkpoint_id"])) + .index(IndexSpec::new( + "by_thread_namespace", + ["thread", "namespace", "seq"], + )), CollectionSpec::new(&self.threads), CollectionSpec::new(&self.writes) .index(IndexSpec::new("by_thread", ["thread"])), @@ -174,7 +192,7 @@ impl DriverCheckpointer { self.declared().await?; let query = Query::filter(Filter::eq("thread", thread)) .sort(Sort::asc("seq")) - .limit(500); + .limit(PAGE); self.docs .query_all(&self.checkpoints, &query) .await @@ -182,8 +200,7 @@ impl DriverCheckpointer { } fn writes_id(thread: &str, namespace: &[String], checkpoint_id: &str) -> String { - let namespace = namespace.join("\u{1f}"); - key(&[thread, &namespace, checkpoint_id]) + key(&[thread, &namespace_key(namespace), checkpoint_id]) } async fn drop_writes(&self, filter: Filter) -> Result<()> { @@ -221,6 +238,7 @@ where let seq = self.next_seq(&checkpoint.thread_id).await?; let doc = json!({ "thread": checkpoint.thread_id, + "namespace": namespace_key(&checkpoint.namespace), "seq": seq, "checkpoint_id": checkpoint.checkpoint_id, "record": serde_json::to_value(&checkpoint)?, @@ -252,6 +270,30 @@ where page.items.into_iter().next().map(Self::decode).transpose() } + /// One indexed query on the stored namespace, so a parent run and a + /// subgraph sharing a thread never load each other's checkpoints, even + /// when they reuse a checkpoint id. + async fn get_scoped( + &self, + thread_id: &str, + checkpoint_id: Option<&str>, + namespace: &[String], + ) -> Result>> { + self.declared().await?; + let mut filter = + Filter::eq("thread", thread_id).and(Filter::eq("namespace", namespace_key(namespace))); + if let Some(id) = checkpoint_id { + filter = filter.and(Filter::eq("checkpoint_id", id)); + } + let query = Query::filter(filter).sort(Sort::desc("seq")).limit(1); + let page = self + .docs + .query(&self.checkpoints, &query) + .await + .map_err(map_error)?; + page.items.into_iter().next().map(Self::decode).transpose() + } + async fn list(&self, thread_id: &str) -> Result> { Ok(self .get_thread(thread_id) @@ -273,7 +315,7 @@ where self.declared().await?; let counters = self .docs - .query_all(&self.threads, &Query::all().limit(500)) + .query_all(&self.threads, &Query::all().limit(PAGE)) .await .map_err(map_error)?; let mut threads = Vec::new(); From c454bd9335b620719550260ca302c7cdd6553cd5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:32:15 +0300 Subject: [PATCH 36/56] test(checkpoint): cover namespace encoding and scoped read isolation Add tests asserting that namespace keys encode injectively, that scoped reads stay within their namespace when checkpoint ids repeat, and that pending writes do not merge across lookalike namespaces. Also tighten the long-key hash assertion to the exact expected length and verify the hash covers the length-prefixed tuple. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 1 + .../src/checkpoint/drivers_tests.rs | 64 ++++++++++++++++++- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1a7a1eed7..99176854b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1717,6 +1717,7 @@ dependencies = [ "rusqlite", "serde", "serde_json", + "sha2", "tempfile", "tinyagents-harness", "tinyinference-llm", diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs index cdf1881c6..df1df4f05 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -111,8 +111,70 @@ fn long_keys_are_hashed_and_tuples_never_collide() { assert_ne!(key(&["a/b", "c"]), key(&["a", "b/c"])); let long = "t".repeat(500); let hashed = key(&[&long]); - assert!(hashed.starts_with("h:") && hashed.len() < 64, "{hashed}"); + assert!(hashed.starts_with("h:") && hashed.len() == 66, "{hashed}"); assert_ne!(hashed, key(&[&"u".repeat(500)])); + assert_ne!( + key(&[&"t".repeat(500)]), + key(&[&"t".repeat(499), "t"]), + "the hash covers the length-prefixed tuple" + ); +} + +#[test] +fn namespaces_encode_injectively() { + let ns = |parts: &[&str]| parts.iter().map(|p| (*p).to_string()).collect::>(); + assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a\u{1f}b"]))); + assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a/b"]))); + assert_ne!(namespace_key(&ns(&["ab"])), namespace_key(&ns(&["a", "b"]))); + assert_eq!(namespace_key(&[]), ""); +} + +#[tokio::test] +async fn scoped_reads_stay_in_their_namespace_when_ids_repeat() { + let saver = checkpointer(); + let child = vec!["sub".to_string()]; + saver.put(sample("t", "same", None, 1)).await.unwrap(); + saver + .put(sample("t", "same", None, 2).with_namespace(child.clone())) + .await + .unwrap(); + let root = saver.get_scoped("t", Some("same"), &[]).await.unwrap().unwrap(); + assert_eq!(root.state, 1, "the root never loads the subgraph's checkpoint"); + let nested = saver + .get_scoped("t", Some("same"), &child) + .await + .unwrap() + .unwrap(); + assert_eq!(nested.state, 2); + assert_eq!(saver.get_scoped("t", None, &[]).await.unwrap().unwrap().state, 1); + assert!( + saver + .get_scoped("t", None, &["other".to_string()]) + .await + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn pending_writes_never_merge_across_lookalike_namespaces() { + let saver = checkpointer(); + let config = |namespace: Vec| CheckpointConfig { + thread_id: "t".to_string(), + checkpoint_id: Some("c".to_string()), + namespace, + }; + let split = config(vec!["a".to_string(), "b".to_string()]); + let joined = config(vec!["a\u{1f}b".to_string()]); + saver + .put_writes( + &split, + &[PendingWrite::data("n", "task", 0, "out", json!("split"))], + ) + .await + .unwrap(); + assert!(saver.get_writes(&joined).await.unwrap().is_empty()); + assert_eq!(saver.get_writes(&split).await.unwrap().len(), 1); } #[test] From 0b0a934c41e9c20c0eeb750424178af822b0966d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:33:10 +0300 Subject: [PATCH 37/56] fix(store): disambiguate prefixed stream names with prefix length Prefixed append stores now name driver streams `:` instead of simply concatenating the prefix, so distinct prefix and stream pairs such as `("a", "bc")` and `("ab", "c")` can no longer collide on the same backend stream. An empty prefix still maps to the bare stream name, and a test covers the previously overlapping case. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-harness/src/store/drivers.rs | 17 ++++++++++++----- .../src/store/drivers_tests.rs | 4 ++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/tinyagents-harness/src/store/drivers.rs b/crates/tinyagents-harness/src/store/drivers.rs index f5b922d3f..c121d5dbc 100644 --- a/crates/tinyagents-harness/src/store/drivers.rs +++ b/crates/tinyagents-harness/src/store/drivers.rs @@ -14,8 +14,10 @@ //! `{ "ns": , "key": , "value": }`, so a namespace //! or key may hold any characters and `list` is an indexed query on `ns`. //! - [`DriverAppendStore`] maps each harness stream to a driver stream of the -//! same name, optionally prefixed. Offsets are the driver's dense offsets, -//! which already match the [`AppendStore`] contract. +//! same name, or with a prefix to `:`; the length +//! keeps two prefixes from ever addressing one stream. Offsets are the +//! driver's dense offsets, which already match the [`AppendStore`] +//! contract. use std::sync::Arc; @@ -160,8 +162,9 @@ impl DriverAppendStore { Self::with_prefix(streams, "") } - /// Map each harness stream to ``, so several independent - /// append stores can share one backend. + /// Map each harness stream to `:`, so several + /// independent append stores can share one backend. The prefix length is + /// part of the name, so `("a", "bc")` and `("ab", "c")` stay apart. pub fn with_prefix(streams: Arc, prefix: impl Into) -> Self { Self { streams, @@ -170,7 +173,11 @@ impl DriverAppendStore { } fn name(&self, stream: &str) -> String { - format!("{}{stream}", self.prefix) + if self.prefix.is_empty() { + stream.to_owned() + } else { + format!("{}:{}{stream}", self.prefix.len(), self.prefix) + } } } diff --git a/crates/tinyagents-harness/src/store/drivers_tests.rs b/crates/tinyagents-harness/src/store/drivers_tests.rs index 1ca786b6e..2953e4f03 100644 --- a/crates/tinyagents-harness/src/store/drivers_tests.rs +++ b/crates/tinyagents-harness/src/store/drivers_tests.rs @@ -113,6 +113,10 @@ async fn read_from_follows_long_streams_and_prefixes_isolate() { assert_eq!(other.len("s").await.unwrap(), 0); let error = journal.append("", json!(1)).await; assert!(error.is_ok(), "a prefix makes the empty stream name valid"); + let ab = DriverAppendStore::with_prefix(Arc::clone(&streams), "ab"); + let a = DriverAppendStore::with_prefix(Arc::clone(&streams), "a"); + ab.append("c", json!("ab/c")).await.unwrap(); + assert_eq!(a.len("bc").await.unwrap(), 0, "prefixes never overlap"); let bare = DriverAppendStore::new(Arc::clone(scoped(&storage, "local").streams())); let error = bare.append("", json!(1)).await.unwrap_err(); assert!(matches!(error, TinyAgentsError::Validation(_)), "{error:?}"); From 41549efe28101aa1823d89976f45ba3cf5bc8783 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:34:27 +0300 Subject: [PATCH 38/56] test: reformat assertions and clone shared streams handle Reformatted long assertions in the checkpoint driver tests to satisfy rustfmt, and cloned the shared streams handle before passing it to a second prefixed store so the first store keeps ownership. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/checkpoint/drivers_tests.rs | 31 ++++++++++++++++--- .../src/store/drivers_tests.rs | 2 +- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs index df1df4f05..2a8ee4d35 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers_tests.rs @@ -123,8 +123,14 @@ fn long_keys_are_hashed_and_tuples_never_collide() { #[test] fn namespaces_encode_injectively() { let ns = |parts: &[&str]| parts.iter().map(|p| (*p).to_string()).collect::>(); - assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a\u{1f}b"]))); - assert_ne!(namespace_key(&ns(&["a", "b"])), namespace_key(&ns(&["a/b"]))); + assert_ne!( + namespace_key(&ns(&["a", "b"])), + namespace_key(&ns(&["a\u{1f}b"])) + ); + assert_ne!( + namespace_key(&ns(&["a", "b"])), + namespace_key(&ns(&["a/b"])) + ); assert_ne!(namespace_key(&ns(&["ab"])), namespace_key(&ns(&["a", "b"]))); assert_eq!(namespace_key(&[]), ""); } @@ -138,15 +144,30 @@ async fn scoped_reads_stay_in_their_namespace_when_ids_repeat() { .put(sample("t", "same", None, 2).with_namespace(child.clone())) .await .unwrap(); - let root = saver.get_scoped("t", Some("same"), &[]).await.unwrap().unwrap(); - assert_eq!(root.state, 1, "the root never loads the subgraph's checkpoint"); + let root = saver + .get_scoped("t", Some("same"), &[]) + .await + .unwrap() + .unwrap(); + assert_eq!( + root.state, 1, + "the root never loads the subgraph's checkpoint" + ); let nested = saver .get_scoped("t", Some("same"), &child) .await .unwrap() .unwrap(); assert_eq!(nested.state, 2); - assert_eq!(saver.get_scoped("t", None, &[]).await.unwrap().unwrap().state, 1); + assert_eq!( + saver + .get_scoped("t", None, &[]) + .await + .unwrap() + .unwrap() + .state, + 1 + ); assert!( saver .get_scoped("t", None, &["other".to_string()]) diff --git a/crates/tinyagents-harness/src/store/drivers_tests.rs b/crates/tinyagents-harness/src/store/drivers_tests.rs index 2953e4f03..8a6465db4 100644 --- a/crates/tinyagents-harness/src/store/drivers_tests.rs +++ b/crates/tinyagents-harness/src/store/drivers_tests.rs @@ -103,7 +103,7 @@ async fn read_from_follows_long_streams_and_prefixes_isolate() { let storage = MemoryStorage::new(); let streams = Arc::clone(scoped(&storage, "local").streams()); let journal = DriverAppendStore::with_prefix(Arc::clone(&streams), "journal/"); - let other = DriverAppendStore::with_prefix(streams, "other/"); + let other = DriverAppendStore::with_prefix(Arc::clone(&streams), "other/"); for i in 0..(READ_PAGE as u64 + 5) { journal.append("s", json!(i)).await.unwrap(); } From 6b8c80633fbb35c05a49f10e7eac7488186414c5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:36:50 +0300 Subject: [PATCH 39/56] fix(session): hash oversized transcript doc keys with sha-256 Oversized document ids were hashed with FNV-1a, which is not collision resistant and could let distinct keys map to the same document. They are now hashed with SHA-256, and the journal stream prefix is documented as length-prefixed by the harness append store. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/port/drivers/mod.rs | 2 +- .../src/port/drivers/transcripts.rs | 11 +++-------- .../src/port/drivers/transcripts_tests.rs | 2 +- docs/modules/session/store-port.md | 2 +- 4 files changed, 6 insertions(+), 11 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index 457d6a156..715b8fe27 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -14,7 +14,7 @@ //! | transcripts | `session_transcripts` (one index document per stem) and `session_transcript_entries` (an append-only log per stem) | //! | turn states | `session_turn_states`, one document per turn | //! | key-value | `session_kv`, through the harness [`DriverStore`] | -//! | journal | streams prefixed `session_journal/`, through the harness [`DriverAppendStore`] | +//! | journal | streams `16:session_journal/`, through the harness [`DriverAppendStore`] | //! //! # Sync seams //! diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index 872ba55d8..2958a438b 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -62,14 +62,8 @@ const STALE_RESERVATION_MS: i64 = 30_000; /// an entry's `#` suffix. const MAX_KEY_LEN: usize = 400; -fn fnv1a(bytes: &[u8]) -> u64 { - bytes.iter().fold(0xcbf2_9ce4_8422_2325, |hash, byte| { - (hash ^ u64::from(*byte)).wrapping_mul(0x0100_0000_01b3) - }) -} - /// A document id for `parts`: length-prefixed so no two tuples collide, and -/// hashed when it would exceed [`MAX_KEY_LEN`]. +/// replaced by its SHA-256 when it would exceed [`MAX_KEY_LEN`]. pub(super) fn doc_key(parts: &[&str]) -> String { let joined: String = parts .iter() @@ -79,7 +73,8 @@ pub(super) fn doc_key(parts: &[&str]) -> String { if joined.len() <= MAX_KEY_LEN { joined } else { - format!("h:{:016x}:{}", fnv1a(joined.as_bytes()), joined.len()) + use sha2::{Digest, Sha256}; + format!("h:{}", hex::encode(Sha256::digest(joined.as_bytes()))) } } diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 6292ab96e..6b524521a 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -393,7 +393,7 @@ fn ids_are_unambiguous_and_bounded() { assert_ne!(doc_key(&["a/b", "c"]), doc_key(&["a", "b/c"])); let long = "x".repeat(1_000); let hashed = doc_key(&[&long]); - assert!(hashed.starts_with("h:")); + assert!(hashed.starts_with("h:") && hashed.len() == 66, "{hashed}"); assert!(entry_id(&long, 9).len() < tinystoragedrivers_core::MAX_ID_LEN); assert_eq!(entry_id("s", 7), "1:s#0000000007"); assert!(is_subagent("parent__child")); diff --git a/docs/modules/session/store-port.md b/docs/modules/session/store-port.md index 6afc6552e..2241b9bf2 100644 --- a/docs/modules/session/store-port.md +++ b/docs/modules/session/store-port.md @@ -82,7 +82,7 @@ provider claims isolation and passes `session_store_isolation_conformance`. | transcripts | `session_transcripts`: one index document per stem (thread, agent, sub-agent flag, creation time); `session_transcript_entries`: an append-only log per stem | | turn states | `session_turn_states`: one document per `(thread, request)` | | key-value | `session_kv` through the harness `DriverStore` | -| journal | streams `session_journal/` through the harness `DriverAppendStore` | +| journal | streams `16:session_journal/` (the harness `DriverAppendStore` length-prefixes its prefix) | - **Transcript log.** Each write is one entry, numbered from 0 and claimed with an insert-only write. An ordinary turn stores only its new rows From ae6713a088c326ffcf9069bddf3e18d354d118d9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:55:02 +0300 Subject: [PATCH 40/56] refactor(transcript): extract stamped line building for reuse The per-message stamping logic in serialise_message_lines is pulled into a stamped_lines helper, and a new stamped_rows function exposes the same provenance (usage, request ids, step stamps) as transcript messages so non-file backends can replay turns identically. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/transcript.rs | 1 + .../src/transcript/jsonl.rs | 69 ++++++++++++++----- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/crates/tinyagents-session/src/transcript.rs b/crates/tinyagents-session/src/transcript.rs index 437f023e5..2a78f99a8 100644 --- a/crates/tinyagents-session/src/transcript.rs +++ b/crates/tinyagents-session/src/transcript.rs @@ -152,6 +152,7 @@ mod writer; pub use adoption::{SessionAdoption, adopt_legacy_session_transcripts}; pub use background::{append_background_message, append_background_message_with_lock_notification}; pub(crate) use history::same_transcript_messages; +pub(crate) use jsonl::stamped_rows; pub use history::{ FileTranscriptHistory, FileTranscriptLocator, TranscriptHistory, TranscriptLocator, TranscriptPartial, TranscriptRead, TranscriptTurn, TruncateCut, diff --git a/crates/tinyagents-session/src/transcript/jsonl.rs b/crates/tinyagents-session/src/transcript/jsonl.rs index cc935dfdf..9cef1ba73 100644 --- a/crates/tinyagents-session/src/transcript/jsonl.rs +++ b/crates/tinyagents-session/src/transcript/jsonl.rs @@ -616,23 +616,10 @@ pub(super) fn serialise_message_lines( request_id: Option<&str>, buf: &mut String, ) -> Result<()> { - let last_assistant_idx = messages.iter().rposition(|m| m.role == "assistant"); - let turn_stamps = turn_step_stamps(messages, last_assistant_idx, last_assistant_turn_usage); - for (i, msg) in messages.iter().enumerate() { - let turn_usage = if Some(i) == last_assistant_idx { - last_assistant_turn_usage - .cloned() - .or_else(|| msg.turn_usage.clone()) - } else { - msg.turn_usage.clone() - }; - let mut line = build_message_line(msg, turn_usage.as_ref(), request_id, false); - if let Some((iteration, ts)) = turn_stamps.get(&i) { - line.iteration = line.iteration.or(Some(*iteration)); - if line.ts.is_none() && !ts.is_empty() { - line.ts = Some(ts.clone()); - } - } + for (i, line) in stamped_lines(messages, last_assistant_turn_usage, request_id) + .into_iter() + .enumerate() + { let line_json = serde_json::to_string(&line).with_context(|| format!("serialise message line {i}"))?; buf.push_str(&line_json); @@ -641,6 +628,54 @@ pub(super) fn serialise_message_lines( Ok(()) } +/// The lines [`serialise_message_lines`] writes for `messages`: the turn's +/// usage on its last assistant row, `request_id` on every fresh row, and the +/// per-step `(iteration, ts)` stamps. +fn stamped_lines( + messages: &[TranscriptMessage], + last_assistant_turn_usage: Option<&TurnUsage>, + request_id: Option<&str>, +) -> Vec { + let last_assistant_idx = messages.iter().rposition(|m| m.role == "assistant"); + let turn_stamps = turn_step_stamps(messages, last_assistant_idx, last_assistant_turn_usage); + messages + .iter() + .enumerate() + .map(|(i, msg)| { + let turn_usage = if Some(i) == last_assistant_idx { + last_assistant_turn_usage + .cloned() + .or_else(|| msg.turn_usage.clone()) + } else { + msg.turn_usage.clone() + }; + let mut line = build_message_line(msg, turn_usage.as_ref(), request_id, false); + if let Some((iteration, ts)) = turn_stamps.get(&i) { + line.iteration = line.iteration.or(Some(*iteration)); + if line.ts.is_none() && !ts.is_empty() { + line.ts = Some(ts.clone()); + } + } + line + }) + .collect() +} + +/// `messages` exactly as the JSONL writer would record them for one turn and +/// a reader would return them: what a non-file transcript backend stores so +/// its replay carries the same per-turn provenance (usage, request ids, +/// step stamps) as a transcript file. +pub(crate) fn stamped_rows( + messages: &[TranscriptMessage], + last_assistant_turn_usage: Option<&TurnUsage>, + request_id: Option<&str>, +) -> Vec { + stamped_lines(messages, last_assistant_turn_usage, request_id) + .into_iter() + .map(message_from_line) + .collect() +} + /// Per-step `(iteration, ts)` stamps for the intermediate assistant rows of the /// turn being written. /// From 7f100203062dbfaf7c4527073a52e36e2ba29534 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:55:08 +0300 Subject: [PATCH 41/56] refactor(session): move transcript jsonl handling into its own module Split the JSONL read and write logic out of transcript.rs into a dedicated transcript/jsonl.rs module. This keeps the transcript types separate from the serialization details and makes the file easier to navigate as more formats are added. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-session/src/transcript.rs | 1 + crates/tinyagents-session/src/transcript/jsonl.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/tinyagents-session/src/transcript.rs b/crates/tinyagents-session/src/transcript.rs index 2a78f99a8..949eada9c 100644 --- a/crates/tinyagents-session/src/transcript.rs +++ b/crates/tinyagents-session/src/transcript.rs @@ -152,6 +152,7 @@ mod writer; pub use adoption::{SessionAdoption, adopt_legacy_session_transcripts}; pub use background::{append_background_message, append_background_message_with_lock_notification}; pub(crate) use history::same_transcript_messages; +#[cfg(feature = "storage-drivers")] pub(crate) use jsonl::stamped_rows; pub use history::{ FileTranscriptHistory, FileTranscriptLocator, TranscriptHistory, TranscriptLocator, diff --git a/crates/tinyagents-session/src/transcript/jsonl.rs b/crates/tinyagents-session/src/transcript/jsonl.rs index 9cef1ba73..2acc4de9c 100644 --- a/crates/tinyagents-session/src/transcript/jsonl.rs +++ b/crates/tinyagents-session/src/transcript/jsonl.rs @@ -665,6 +665,7 @@ fn stamped_lines( /// a reader would return them: what a non-file transcript backend stores so /// its replay carries the same per-turn provenance (usage, request ids, /// step stamps) as a transcript file. +#[cfg(feature = "storage-drivers")] pub(crate) fn stamped_rows( messages: &[TranscriptMessage], last_assistant_turn_usage: Option<&TurnUsage>, From 9720092f18b7f618b16fe1cd987f6c81f8b32c03 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:56:10 +0300 Subject: [PATCH 42/56] feat(session): add transcript driver port Introduce a port trait for transcript storage drivers so session code can depend on an abstraction rather than a concrete backend. Tests cover the new trait's contract. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts.rs | 225 ++++++++++++++---- .../src/port/drivers/transcripts_tests.rs | 2 +- 2 files changed, 175 insertions(+), 52 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index 2958a438b..bf65ba74b 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -34,14 +34,15 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use tinystoragedrivers_core::{ Blocking, CollectionSpec, DocumentStore, DocumentStoreExt, ErrorKind, Filter, IndexSpec, - Precondition, Query, Sort, StorageError, + Precondition, Query, Sort, StorageError, Version, }; use tokio::sync::{Mutex, OnceCell}; use super::super::memory::MAX_GENERATIONS; use crate::transcript::{ SessionRef, SessionTranscript, TranscriptHistory, TranscriptLocator, TranscriptMessage, - TranscriptMeta, TranscriptPartial, TranscriptRead, TranscriptTurn, session_stem, + TranscriptMeta, TranscriptPartial, TranscriptRead, TranscriptTurn, TurnUsage, + same_transcript_messages, session_stem, stamped_rows, }; /// One document per transcript stem: the lookup fields. @@ -49,6 +50,9 @@ pub(super) const INDEX: &str = "session_transcripts"; /// One document per write to a transcript: its append-only log. pub(super) const ENTRIES: &str = "session_transcript_entries"; +/// Entries read per driver round trip while replaying a log. +const PAGE: usize = 500; + /// Insert attempts before a contended write gives up. const CAS_ATTEMPTS: usize = 64; @@ -296,8 +300,25 @@ impl DriverTranscriptHistory { } /// Seals this generation; later writes to it are refused. - fn seal(&self) -> anyhow::Result<()> { - self.commit(|replay, _| { + /// + /// With a `baseline`, the seal is conditional on the transcript still + /// holding exactly those messages, checked against the same replay the + /// seal is claimed on: a turn another writer committed after the caller + /// read the baseline makes the seal fail instead of being dropped from + /// the successor. + fn seal(&self, baseline: Option>) -> anyhow::Result<()> { + self.commit(move |replay, _| { + if let Some(baseline) = &baseline { + let current: &[TranscriptMessage] = if replay.written { + &replay.messages + } else { + &[] + }; + anyhow::ensure!( + same_transcript_messages(current, baseline), + "transcript baseline is stale; reload the session before creating a generation" + ); + } Ok((!replay.sealed).then(|| Entry { seal: true, ..Entry::default() @@ -338,7 +359,9 @@ impl HistoryInner { Filter::eq("stem", self.stem.as_str()).and(Filter::gte("seq", replay.next_seq)), ) .sort(Sort::asc("seq")) - .limit(500); + .limit(PAGE); + // `query_all` follows the cursors: `PAGE` bounds one round trip, not + // the replay. for stored in self.docs.query_all(ENTRIES, &query).await? { let seq = stored .doc @@ -389,7 +412,18 @@ impl HistoryInner { { Ok(_) => { replay.apply(seq, entry); - self.index(&mut replay).await?; + // The entry is durable: report the write as done. A + // failed index refresh only delays lookups by thread or + // agent until this handle's next write retries it. + if let Err(error) = self.index(&mut replay).await { + tracing::warn!( + target: "tinyagents_session::port::drivers", + stem = %self.stem, + seq, + %error, + "[session-store] transcript index refresh failed; retried on the next write" + ); + } return Ok(true); } Err(error) if error.kind() == ErrorKind::Conflict => {} @@ -404,6 +438,11 @@ impl HistoryInner { /// Refreshes this stem's [`INDEX`] document when its lookup fields /// changed. + /// + /// Fenced by log position: the document records the entry it was built + /// from (`indexed_seq`), and a handle whose replay is older than that + /// leaves it alone, so a delayed writer never puts back stale lookup + /// fields. The write itself is a compare-and-swap on the version read. async fn index(&self, replay: &mut Replay) -> Result<(), StorageError> { if !replay.written { return Ok(()); @@ -420,18 +459,44 @@ impl HistoryInner { if replay.indexed.as_ref() == Some(&fields) { return Ok(()); } + let seq = replay.next_seq.saturating_sub(1); let id = doc_key(&[&self.stem]); - let created_at = self - .docs - .get(INDEX, &id) - .await? - .and_then(|found| found.doc.get("created_at").cloned()) - .unwrap_or_else(|| json!(now_rfc3339())); - let mut doc = fields.clone(); - doc["created_at"] = created_at; - self.docs.put(INDEX, &id, doc, Precondition::None).await?; - replay.indexed = Some(fields); - Ok(()) + for _ in 0..CAS_ATTEMPTS { + let existing = self.docs.get(INDEX, &id).await?; + let newer = existing.as_ref().is_some_and(|found| { + found.doc.get("written") == Some(&json!(true)) + && found + .doc + .get("indexed_seq") + .and_then(Value::as_u64) + .is_some_and(|indexed| indexed > seq) + }); + if newer { + return Ok(()); + } + let created_at = existing + .as_ref() + .and_then(|found| found.doc.get("created_at").cloned()) + .unwrap_or_else(|| json!(now_rfc3339())); + let precondition = existing + .as_ref() + .map_or(Precondition::Absent, |found| found.unchanged()); + let mut doc = fields.clone(); + doc["created_at"] = created_at; + doc["indexed_seq"] = json!(seq); + match self.docs.put(INDEX, &id, doc, precondition).await { + Ok(_) => { + replay.indexed = Some(fields); + return Ok(()); + } + Err(error) if error.kind() == ErrorKind::Conflict => {} + Err(error) => return Err(error), + } + } + Err(StorageError::conflict(format!( + "transcript index for {} kept changing under {CAS_ATTEMPTS} attempts", + self.stem + ))) } } @@ -454,18 +519,35 @@ impl TranscriptRead for DriverTranscriptHistory { } } -/// The entry recording a turn whose logical messages are now `next`: only -/// the new rows when `next` extends what is stored, all of them otherwise. +/// The entry recording a turn whose logical messages are now `next`. +/// +/// Mirrors the JSONL writer: the turn's `prev` must be what is stored +/// (otherwise the caller's view is stale and the write is refused rather than +/// allowed to replace newer rows), an extension stores only its new rows and +/// anything else stores the whole set, and the written rows carry the turn's +/// usage and request id exactly as a transcript file records them. fn turn_entry(replay: &Replay, turn: &TurnRecord) -> anyhow::Result> { anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); - let stored = &replay.messages; - let extends = replay.written - && turn.next.len() >= stored.len() - && turn.next[..stored.len()] == stored[..]; - let (set, extend) = if extends { - (None, Some(turn.next[stored.len()..].to_vec())) + let stored: &[TranscriptMessage] = if replay.written { + &replay.messages + } else { + &[] + }; + anyhow::ensure!( + !replay.written || same_transcript_messages(stored, &turn.prev), + "transcript baseline is stale; reload the session before persisting" + ); + let common = stored + .iter() + .zip(&turn.next) + .take_while(|(left, right)| left.same_row_as(right)) + .count(); + let usage = turn.turn_usage.as_ref(); + let request_id = turn.request_id.as_deref(); + let (set, extend) = if replay.written && common == stored.len() { + (None, Some(stamped_rows(&turn.next[common..], usage, request_id))) } else { - (Some(turn.next.clone()), None) + (Some(stamped_rows(&turn.next, usage, request_id)), None) }; Ok(Some(Entry { written: true, @@ -481,8 +563,10 @@ fn turn_entry(replay: &Replay, turn: &TurnRecord) -> anyhow::Result, next: Vec, meta: TranscriptMeta, + turn_usage: Option, tools: Option, request_id: Option, partial: Option, @@ -491,8 +575,10 @@ struct TurnRecord { impl TurnRecord { fn new(turn: &TranscriptTurn<'_>, partial: Option<&TranscriptPartial>) -> Self { Self { + prev: normalized(turn.prev.to_vec()), next: normalized(turn.next.to_vec()), meta: turn.meta.clone(), + turn_usage: turn.turn_usage.cloned(), tools: turn.tools.cloned(), request_id: turn.request_id.map(str::to_string), partial: partial @@ -678,7 +764,7 @@ impl DriverTranscriptLocator { /// Reserves `stem` as a new generation: succeeds when nothing is written /// there and no live reservation holds it. - async fn reserve(docs: &Arc, stem: &str) -> anyhow::Result<()> { + async fn reserve(docs: &Arc, stem: &str) -> anyhow::Result { let written = docs .count( ENTRIES, @@ -709,7 +795,7 @@ impl DriverTranscriptLocator { } }; match docs.put(INDEX, &id, reservation, precondition).await { - Ok(_) => Ok(()), + Ok(version) => Ok(version), Err(error) if error.kind() == ErrorKind::Conflict => { anyhow::bail!("session generation already exists or is reserved") } @@ -846,6 +932,29 @@ impl TranscriptLocator for DriverTranscriptLocator { &self, session: &SessionRef, seed: TranscriptMeta, + ) -> anyhow::Result<(SessionRef, Arc)> { + self.begin_generation_checked(session, seed, None) + } + + /// [`Self::begin_generation`], sealing only if the predecessor still + /// holds exactly `baseline` — checked as part of the seal itself, so a + /// turn committed in between cannot be lost from the successor. + fn begin_generation_from_baseline( + &self, + session: &SessionRef, + seed: TranscriptMeta, + baseline: &[TranscriptMessage], + ) -> anyhow::Result<(SessionRef, Arc)> { + self.begin_generation_checked(session, seed, Some(normalized(baseline.to_vec()))) + } +} + +impl DriverTranscriptLocator { + fn begin_generation_checked( + &self, + session: &SessionRef, + seed: TranscriptMeta, + baseline: Option>, ) -> anyhow::Result<(SessionRef, Arc)> { let successor = session.next_generation(); anyhow::ensure!( @@ -853,32 +962,21 @@ impl TranscriptLocator for DriverTranscriptLocator { "session generation limit reached" ); let stem = session_stem(&successor); - let reserved = { + let reservation = { let stem = stem.clone(); - run_on(&self.bridge, { - let docs = Arc::clone(&self.docs); - let declared = Arc::clone(&self.declared); - async move { - declared.ensure(&docs).await?; - Ok(Self::reserve(&docs, &stem).await) - } - })? + let docs = Arc::clone(&self.docs); + let declared = Arc::clone(&self.declared); + run_on(&self.bridge, async move { + declared.ensure(&docs).await?; + Ok(Self::reserve(&docs, &stem).await) + })?? }; - reserved?; let predecessor_stem = session_stem(session); - if let Err(error) = self.handle(&predecessor_stem, seed.clone()).seal() { - let docs = Arc::clone(&self.docs); - let id = doc_key(&[&stem]); - if let Err(release) = run_on(&self.bridge, async move { - docs.delete(INDEX, &id, Precondition::None).await - }) { - tracing::warn!( - target: "tinyagents_session::port::drivers", - stem = %stem, - error = %release, - "[session-store] could not release a generation reservation" - ); - } + if let Err(error) = self + .handle(&predecessor_stem, seed.clone()) + .seal(baseline) + { + self.release(&stem, reservation); return Err(error); } let mut meta = seed; @@ -887,6 +985,31 @@ impl TranscriptLocator for DriverTranscriptLocator { let handle = self.handle(&stem, meta); Ok((successor, Arc::new(handle))) } + + /// Drops the reservation of `stem` this call made, at the version it + /// wrote: if another process has since taken it over or written the + /// successor, the document is theirs and stays. + fn release(&self, stem: &str, reservation: Version) { + let docs = Arc::clone(&self.docs); + let id = doc_key(&[stem]); + let released = run_on(&self.bridge, async move { + match docs + .delete(INDEX, &id, Precondition::Version(reservation)) + .await + { + Err(error) if error.kind() == ErrorKind::Conflict => Ok(false), + other => other, + } + }); + if let Err(error) = released { + tracing::warn!( + target: "tinyagents_session::port::drivers", + stem, + %error, + "[session-store] could not release a generation reservation" + ); + } + } } #[cfg(test)] diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 6b524521a..e1c19b131 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -246,7 +246,7 @@ fn a_sealed_generation_refuses_writes() { .record_partial(&TranscriptPartial::new("x"), None) .unwrap() ); - handle.seal().unwrap(); + handle.seal(None).unwrap(); assert_eq!( entries(&docs, &session_stem(&session)) .iter() From befefc8ce581da8494ec86330434326ad895f065 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:57:12 +0300 Subject: [PATCH 43/56] refactor(session): split driver port tests into focused modules Move the driver port tests out of the monolithic module into separate files covering module wiring, transcripts, and turn states. This keeps each test file scoped to a single concern and makes the port easier to navigate as it grows. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/mod.rs | 98 +++++++++++----- .../src/port/drivers/mod_tests.rs | 18 +++ .../src/port/drivers/transcripts_tests.rs | 111 ++++++++++++++++++ .../src/port/drivers/turn_states.rs | 27 +++-- 4 files changed, 212 insertions(+), 42 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index 715b8fe27..9b9d71347 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -26,8 +26,9 @@ //! # Scopes //! //! An agent id that is a valid [`Scope`] is used as is. Any other id (empty, -//! over-long, or holding whitespace) maps to `sha256:` of the id, so -//! every agent still gets a scope of its own and the mapping never changes. +//! over-long, holding whitespace) and any id that itself starts with +//! `sha256:` maps to `sha256:` of the id, so every agent gets a scope of +//! its own, no raw id can name a hashed scope, and the mapping never changes. mod refused; mod transcripts; @@ -45,6 +46,10 @@ use super::{AgentStores, SessionStoreProvider}; pub use transcripts::{DriverTranscriptHistory, DriverTranscriptLocator}; pub use turn_states::DriverTurnStates; +/// Prefix of the scopes agent ids are hashed into. Reserved: an agent id +/// starting with it is hashed too, so it cannot name another agent's scope. +const HASHED_SCOPE: &str = "sha256:"; + /// Collection holding each agent's key-value records. const KV_COLLECTION: &str = "session_kv"; /// Prefix of each agent's journal streams. @@ -108,11 +113,14 @@ impl DriverSessionStores { /// The scope agent `agent_id` is stored under. pub fn scope_for(agent_id: &str) -> Scope { - Scope::new(agent_id).unwrap_or_else(|_| { - let digest = Sha256::digest(agent_id.as_bytes()); - Scope::new(format!("sha256:{}", hex::encode(digest))) - .expect("a sha256 hex scope is always valid") - }) + if !agent_id.starts_with(HASHED_SCOPE) + && let Ok(scope) = Scope::new(agent_id) + { + return scope; + } + let digest = Sha256::digest(agent_id.as_bytes()); + Scope::new(format!("{HASHED_SCOPE}{}", hex::encode(digest))) + .expect("a sha256 hex scope is always valid") } /// The stores of `agent_id`, or the error that kept the backend from @@ -132,29 +140,9 @@ impl DriverSessionStores { } let scoped = self.backend.for_scope(&Self::scope_for(agent_id))?; let stores = self.build(&scoped); - if self.recover_on_open - && self - .recovered - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(agent_id.to_string()) - { - let now = chrono::Utc::now().to_rfc3339(); - match stores.turn_states.mark_all_interrupted(&now) { - Ok(count) if count > 0 => tracing::info!( - target: "tinyagents_session::port::drivers", - agent_id, - count, - "[session-store] marked turns left in flight interrupted" - ), - Ok(_) => {} - Err(error) => tracing::warn!( - target: "tinyagents_session::port::drivers", - agent_id, - %error, - "[session-store] could not recover in-flight turns" - ), - } + if self.recover_on_open && !self.recover_agent(agent_id, &stores) { + // Not cached, so the next open tries the recovery again. + return Ok(stores); } Ok(self .agents @@ -165,9 +153,57 @@ impl DriverSessionStores { .clone()) } + /// Interrupts `agent_id`'s in-flight turns unless this provider already + /// did; returns whether the agent is now recovered. + /// + /// The `recovered` lock is held across the sweep, so a concurrent first + /// open of the same agent waits for it instead of handing out stores a + /// still-running sweep could interrupt a new turn on. The agent is + /// recorded only once the sweep succeeds. + fn recover_agent(&self, agent_id: &str, stores: &AgentStores) -> bool { + let mut recovered = self + .recovered + .lock() + .unwrap_or_else(PoisonError::into_inner); + if recovered.contains(agent_id) { + return true; + } + let now = chrono::Utc::now().to_rfc3339(); + match stores.turn_states.mark_all_interrupted(&now) { + Ok(count) => { + if count > 0 { + tracing::info!( + target: "tinyagents_session::port::drivers", + agent_id, + count, + "[session-store] marked turns left in flight interrupted" + ); + } + recovered.insert(agent_id.to_string()); + true + } + Err(error) => { + tracing::warn!( + target: "tinyagents_session::port::drivers", + agent_id, + %error, + "[session-store] could not recover in-flight turns; retried on the next open" + ); + false + } + } + } + fn build(&self, scoped: &ScopedStorage) -> AgentStores { let docs = Arc::clone(scoped.documents()); - let label = format!("{}://{}", scoped.driver(), scoped.scope()); + // The backend's address keeps two backends with the same driver and + // scope from claiming one destination. + let label = format!( + "{}://{:p}/{}", + scoped.driver(), + Arc::as_ptr(&self.backend), + scoped.scope() + ); AgentStores { transcripts: Arc::new(DriverTranscriptLocator::new( Arc::clone(&docs), diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs index 2e9589b49..2dc0fc259 100644 --- a/crates/tinyagents-session/src/port/drivers/mod_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/mod_tests.rs @@ -75,6 +75,24 @@ fn any_other_agent_id_maps_to_a_stable_hashed_scope() { ); } +#[test] +fn a_raw_id_can_never_name_a_hashed_scope() { + let hashed = DriverSessionStores::scope_for("has a space"); + let impostor = DriverSessionStores::scope_for(hashed.as_str()); + assert_ne!(impostor, hashed, "ids in the reserved prefix are hashed too"); + assert!(impostor.as_str().starts_with("sha256:")); +} + +#[test] +fn two_backends_never_share_a_destination() { + let one = memory().for_agent("a"); + let two = memory().for_agent("a"); + assert_ne!( + one.transcripts.destination_key(), + two.transcripts.destination_key() + ); +} + #[test] fn an_agent_gets_the_same_stores_every_time() { let provider = memory(); diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index e1c19b131..729b98671 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -459,3 +459,114 @@ fn handles_describe_themselves() { Some("memory://test/transcripts") ); } + +#[test] +fn a_stale_turn_is_refused_instead_of_replacing_newer_rows() { + let docs = docs(); + let locator = locator(&docs); + let one = locator.open_stem("s", meta("t")).unwrap(); + let two = locator.open_stem("s", meta("t")).unwrap(); + let a = vec![message("user", "a")]; + one.append_turn(turn(&[], &a, &meta("t"))).unwrap(); + let mut ab = a.clone(); + ab.push(message("assistant", "b")); + one.append_turn(turn(&a, &ab, &meta("t"))).unwrap(); + // `two` still believes the transcript is `a`. + let mut ac = a.clone(); + ac.push(message("assistant", "c")); + let error = two.append_turn(turn(&a, &ac, &meta("t"))).unwrap_err(); + assert!(error.to_string().contains("stale"), "{error}"); + let contents: Vec = two + .messages() + .unwrap() + .into_iter() + .map(|row| row.content) + .collect(); + assert_eq!(contents, ["a", "b"], "the newer rows survive"); +} + +#[test] +fn a_turn_keeps_its_usage_and_request_id() { + let docs = docs(); + let history = locator(&docs).open_stem("s", meta("t")).unwrap(); + let rows = vec![message("user", "q"), message("assistant", "a")]; + let usage: crate::transcript::TurnUsage = + serde_json::from_value(json!({ "input_tokens": 7, "output_tokens": 3 })).unwrap(); + history + .append_turn(TranscriptTurn { + turn_usage: Some(&usage), + request_id: Some("req-1"), + ..turn(&[], &rows, &meta("t")) + }) + .unwrap(); + let read = history.messages().unwrap(); + assert!(read.iter().all(|row| row.request_id.as_deref() == Some("req-1"))); + assert!(read[1].turn_usage.is_some(), "the assistant row carries the usage"); + assert!(read[0].turn_usage.is_none()); + + // The next turn, built from the rows as read back, extends them. + let mut next = read.clone(); + next.push(message("user", "again")); + history + .append_turn(TranscriptTurn { + request_id: Some("req-2"), + ..turn(&read, &next, &meta("t")) + }) + .unwrap(); + let log = entries(&docs, "s"); + assert_eq!(log[1]["extend"].as_array().unwrap().len(), 1); + let read = history.messages().unwrap(); + assert_eq!(read[0].request_id.as_deref(), Some("req-1"), "old rows keep theirs"); + assert_eq!(read[2].request_id.as_deref(), Some("req-2")); +} + +#[test] +fn a_stale_baseline_fails_the_seal_and_frees_the_successor() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + let history = locator.open_session(&session, meta("t")).unwrap(); + let a = vec![message("user", "a")]; + history.append_turn(turn(&[], &a, &meta("t"))).unwrap(); + let mut ab = a.clone(); + ab.push(message("assistant", "b")); + history.append_turn(turn(&a, &ab, &meta("t"))).unwrap(); + + let error = locator + .begin_generation_from_baseline(&session, meta("t"), &a) + .err() + .unwrap(); + assert!(error.to_string().contains("stale"), "{error}"); + let mut abc = ab.clone(); + abc.push(message("user", "c")); + history + .append_turn(turn(&ab, &abc, &meta("t"))) + .expect("the predecessor was not sealed"); + + let (successor, _) = locator + .begin_generation_from_baseline(&session, meta("t"), &abc) + .expect("the released reservation is free again"); + assert_eq!(successor.generation, 1); +} + +#[test] +fn a_stale_handle_never_rewinds_the_index() { + let docs = docs(); + let locator = locator(&docs); + let rows = vec![message("user", "a")]; + let old = locator.handle("s", meta("old")); + old.append_turn(turn(&[], &rows, &meta("old"))).unwrap(); + let new = locator.open_stem("s", meta("new")).unwrap(); + new.append_turn(turn(&rows, &rows, &meta("new"))).unwrap(); + + // `old` re-indexes from a replay that predates `new`'s entry. + let inner = Arc::clone(&old.inner); + on_bridge(async move { + let mut replay = inner.replay.lock().await; + replay.indexed = None; + replay.next_seq = 1; + inner.index(&mut replay).await + }); + assert!(locator.root_for_thread("new").is_some()); + assert!(locator.root_for_thread("old").is_none()); +} diff --git a/crates/tinyagents-session/src/port/drivers/turn_states.rs b/crates/tinyagents-session/src/port/drivers/turn_states.rs index fa2f492a0..0afcd1ef4 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states.rs @@ -172,22 +172,27 @@ impl Inner { /// Drops `thread_id`'s completed turns beyond the newest /// [`COMPLETED_RETENTION`]. + /// + /// Each delete is conditional on the version the listing saw: a turn + /// rewritten in the meantime is someone else's newer state and stays. async fn prune_completed(&self, thread_id: &str) -> Result<(), StorageError> { - let mut completed: Vec = self - .thread(thread_id) + let mut completed: Vec> = self + .query(Filter::eq("thread_id", thread_id)) .await? .into_iter() - .filter(|turn| turn.lifecycle == TurnLifecycle::Completed) + .filter(|stored| stored.doc.lifecycle == TurnLifecycle::Completed) .collect(); - completed.sort_by(completed_newest_first); + completed.sort_by(|a, b| completed_newest_first(&a.doc, &b.doc)); for stale in completed.iter().skip(COMPLETED_RETENTION) { - self.docs - .delete( - COLLECTION, - &id(&stale.thread_id, &stale.request_id), - Precondition::None, - ) - .await?; + match self + .docs + .delete(COLLECTION, &stale.id, stale.unchanged()) + .await + { + Ok(_) => {} + Err(error) if is_race(&error) => {} + Err(error) => return Err(error), + } } Ok(()) } From 4e194801f2e7e5e2970b28e6b58b55a1a387963e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:58:09 +0300 Subject: [PATCH 44/56] test(transcripts): cover turn usage fields and tighten assertions The transcript tests now build TurnUsage from the full provider, model and usage shape and compare message contents directly, so the assertions match what the driver actually stores. Remaining edits are rustfmt reflows with no behaviour change. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/mod_tests.rs | 5 ++- .../src/port/drivers/transcripts.rs | 10 +++--- .../src/port/drivers/transcripts_tests.rs | 32 +++++++++++++++---- crates/tinyagents-session/src/transcript.rs | 4 +-- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs index 2dc0fc259..e4841b9e6 100644 --- a/crates/tinyagents-session/src/port/drivers/mod_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/mod_tests.rs @@ -79,7 +79,10 @@ fn any_other_agent_id_maps_to_a_stable_hashed_scope() { fn a_raw_id_can_never_name_a_hashed_scope() { let hashed = DriverSessionStores::scope_for("has a space"); let impostor = DriverSessionStores::scope_for(hashed.as_str()); - assert_ne!(impostor, hashed, "ids in the reserved prefix are hashed too"); + assert_ne!( + impostor, hashed, + "ids in the reserved prefix are hashed too" + ); assert!(impostor.as_str().starts_with("sha256:")); } diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index bf65ba74b..29aea491f 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -545,7 +545,10 @@ fn turn_entry(replay: &Replay, turn: &TurnRecord) -> anyhow::Result = history + .messages() + .unwrap() + .into_iter() + .map(|row| row.content) + .collect(); + assert_eq!(contents, ["summary"]); } #[test] @@ -490,8 +496,12 @@ fn a_turn_keeps_its_usage_and_request_id() { let docs = docs(); let history = locator(&docs).open_stem("s", meta("t")).unwrap(); let rows = vec![message("user", "q"), message("assistant", "a")]; - let usage: crate::transcript::TurnUsage = - serde_json::from_value(json!({ "input_tokens": 7, "output_tokens": 3 })).unwrap(); + let usage: crate::transcript::TurnUsage = serde_json::from_value(json!({ + "provider": "p", + "model": "m", + "usage": { "input": 7, "output": 3, "cached_input": 0, "cost_usd": 0.01 }, + })) + .unwrap(); history .append_turn(TranscriptTurn { turn_usage: Some(&usage), @@ -500,8 +510,14 @@ fn a_turn_keeps_its_usage_and_request_id() { }) .unwrap(); let read = history.messages().unwrap(); - assert!(read.iter().all(|row| row.request_id.as_deref() == Some("req-1"))); - assert!(read[1].turn_usage.is_some(), "the assistant row carries the usage"); + assert!( + read.iter() + .all(|row| row.request_id.as_deref() == Some("req-1")) + ); + assert!( + read[1].turn_usage.is_some(), + "the assistant row carries the usage" + ); assert!(read[0].turn_usage.is_none()); // The next turn, built from the rows as read back, extends them. @@ -516,7 +532,11 @@ fn a_turn_keeps_its_usage_and_request_id() { let log = entries(&docs, "s"); assert_eq!(log[1]["extend"].as_array().unwrap().len(), 1); let read = history.messages().unwrap(); - assert_eq!(read[0].request_id.as_deref(), Some("req-1"), "old rows keep theirs"); + assert_eq!( + read[0].request_id.as_deref(), + Some("req-1"), + "old rows keep theirs" + ); assert_eq!(read[2].request_id.as_deref(), Some("req-2")); } diff --git a/crates/tinyagents-session/src/transcript.rs b/crates/tinyagents-session/src/transcript.rs index 949eada9c..403d09d08 100644 --- a/crates/tinyagents-session/src/transcript.rs +++ b/crates/tinyagents-session/src/transcript.rs @@ -152,12 +152,12 @@ mod writer; pub use adoption::{SessionAdoption, adopt_legacy_session_transcripts}; pub use background::{append_background_message, append_background_message_with_lock_notification}; pub(crate) use history::same_transcript_messages; -#[cfg(feature = "storage-drivers")] -pub(crate) use jsonl::stamped_rows; pub use history::{ FileTranscriptHistory, FileTranscriptLocator, TranscriptHistory, TranscriptLocator, TranscriptPartial, TranscriptRead, TranscriptTurn, TruncateCut, }; +#[cfg(feature = "storage-drivers")] +pub(crate) use jsonl::stamped_rows; pub use legacy_md::read_transcript_legacy_md; pub use migration::{TranscriptLayoutMigration, migrate_layout_if_needed}; pub use paths::{find_latest_transcript, resolve_keyed_transcript_path}; From 53d97bb1ad74abdbbb7a29ee4b40b73f6ec08ec5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Wed, 7 Oct 2026 15:58:29 +0300 Subject: [PATCH 45/56] docs(session): document store port isolation and write semantics Expand the store port documentation to cover reserved scope prefixes, stale-turn refusal, index compare-and-swap, generation baseline checks, and recovery gating. These details reflect behaviour that was previously undocumented, so operators can reason about isolation and failure modes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/modules/session/store-port.md | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/modules/session/store-port.md b/docs/modules/session/store-port.md index 2241b9bf2..481c4774e 100644 --- a/docs/modules/session/store-port.md +++ b/docs/modules/session/store-port.md @@ -73,8 +73,9 @@ let backend: Arc = Arc::new(SqliteStorage::open(path)?); let provider = DriverSessionStores::new(backend)?.recover_on_open(true); ``` -Each agent id becomes a storage `Scope` (an id that is not a valid scope -maps to `sha256:` of itself), and the driver enforces scopes, so the +Each agent id becomes a storage `Scope`. An id that is not a valid scope, +or that starts with the reserved `sha256:` prefix, maps to `sha256:` of +itself, so no raw id can name a hashed scope, and the driver enforces scopes, so the provider claims isolation and passes `session_store_isolation_conformance`. | Store | Backend shape | @@ -89,19 +90,32 @@ provider claims isolation and passes `session_store_isolation_conformance`. (`extend`); a first write or a compaction stores the whole set (`set`). Two writers on one transcript (two processes on one database) serialize: the loser re-reads and decides again, which is also how a write racing a seal is - refused. + refused. As in the JSONL writer, a turn whose `prev` is not what is stored + is refused as stale rather than allowed to replace newer rows, and the + written rows carry the turn's usage, request id and step stamps, built by + the writer's own code (`transcript::stamped_rows`). +- **Index.** Each index document records the log entry it was built from + (`indexed_seq`). A handle with an older replay leaves it alone, and the + update is a compare-and-swap. A failed index refresh never fails the write + that preceded it; the next write retries it. - **Generations.** `begin_generation` reserves the successor in the index before sealing the predecessor, so two compactions cannot both open it. A reservation left unwritten for 30 seconds (its process stopped mid-way) may be taken over, so a sealed head never strands the conversation. -- **Turn states.** Conditional writes, settling and the interruption sweep - are compare-and-swap on the document version. + `begin_generation_from_baseline` checks the baseline inside the seal + itself, so a turn committed after the caller read it fails the compaction + instead of vanishing from the successor. A failed seal releases the + reservation only at the version this call wrote. +- **Turn states.** Conditional writes, settling, the interruption sweep and + retention pruning are all conditional on the document version. - **Sync seams.** Transcript and turn-state calls run on a `tinystoragedrivers` `Blocking` bridge (one dedicated runtime thread), so they work from any caller, inside a runtime or not. - **Recovery.** A backend cannot list its scopes, so `recover` covers the agents this provider has opened. `recover_on_open(true)` also interrupts an - agent's in-flight turns the first time it is opened. Use it only when a + agent's in-flight turns the first time it is opened. The agent counts as + recovered only once the sweep succeeds, and a concurrent first open waits + for that sweep. Use it only when a single process owns the database (the desktop app). - **Failing closed.** If the backend cannot bind an agent's scope, `for_agent` returns stores that refuse every call with that error and does not cache From 2a4ebff89d43798dd8d4e592825c217555fde324 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:38:38 +0300 Subject: [PATCH 46/56] chore(vendor): bump tinyinference and tinytools submodules Update the vendored tinyinference and tinytools submodule pointers to their latest commits. Auto-committed-on: dragonfly Co-authored-by: Medulla --- vendor/tinyinference | 2 +- vendor/tinytools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/tinyinference b/vendor/tinyinference index a32eec813..fdffc2a45 160000 --- a/vendor/tinyinference +++ b/vendor/tinyinference @@ -1 +1 @@ -Subproject commit a32eec8138d7be0978dd232ec9f93af33945dbe6 +Subproject commit fdffc2a454af2b04e5052eb38dd3add8a6db1b85 diff --git a/vendor/tinytools b/vendor/tinytools index e2bf1be8c..8e5008c37 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit e2bf1be8cff188cfaadcf1b3a3339da8b3e746c4 +Subproject commit 8e5008c375f9e717e6cbb81fa9532599b98ff0a8 From e0a292dc768d964f213eb03be59fbe15e379d315 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:39:23 +0300 Subject: [PATCH 47/56] chore(deps): drop sha2 from lockfile Remove the sha2 entry from the dependency list in Cargo.lock, reflecting that the crate is no longer a dependency. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index cdd254dd5..a9cf3ccc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1949,7 +1949,6 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2", "thiserror", "tinyinference-core", "tinytools-agent", From 648e77a12124b0c23d331a1bf2a4ac472ffbdea0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:39:40 +0300 Subject: [PATCH 48/56] chore(vendor): restore main's tinytools and tinyinference pins A checkpoint commit on this branch had recorded stale gitlinks, so the merge of main kept them and the harness no longer built against tinytools. Co-authored-by: Medulla --- vendor/tinyinference | 2 +- vendor/tinytools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/tinyinference b/vendor/tinyinference index fdffc2a45..a32eec813 160000 --- a/vendor/tinyinference +++ b/vendor/tinyinference @@ -1 +1 @@ -Subproject commit fdffc2a454af2b04e5052eb38dd3add8a6db1b85 +Subproject commit a32eec8138d7be0978dd232ec9f93af33945dbe6 diff --git a/vendor/tinytools b/vendor/tinytools index 8e5008c37..e2bf1be8c 160000 --- a/vendor/tinytools +++ b/vendor/tinytools @@ -1 +1 @@ -Subproject commit 8e5008c375f9e717e6cbb81fa9532599b98ff0a8 +Subproject commit e2bf1be8cff188cfaadcf1b3a3339da8b3e746c4 From 439a2466a6f5d3df9a330c95962172838cad8aa6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:41:59 +0300 Subject: [PATCH 49/56] chore(deps): add sha2 to lockfile Record sha2 in Cargo.lock now that a crate in the workspace depends on it. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index a9cf3ccc2..cdd254dd5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1949,6 +1949,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror", "tinyinference-core", "tinytools-agent", From 593c969f62d9e836ff38bce323c5ab9b33217fe5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:43:38 +0300 Subject: [PATCH 50/56] fix(session): correct transcript replay freshness check The replay now treats an indexed sequence equal to the current one as already fresh and records the indexed fields, so a replayed entry is not re-indexed when the stored document is at least as new. Driver-backed queries also drop their fixed page limits and rely on cursor-following `query_all`, and the storage checkpointer decodes records through the shared classifier so schema mismatches are tagged consistently. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/checkpoint/drivers.rs | 21 +++++++------------ .../tinyagents-harness/src/store/drivers.rs | 17 +++++++-------- .../src/port/drivers/transcripts.rs | 13 +++++------- .../src/port/drivers/turn_states.rs | 2 +- 4 files changed, 21 insertions(+), 32 deletions(-) diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs index d6d154f69..7cbad5acb 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -39,7 +39,7 @@ use tokio::sync::OnceCell; use super::{ Checkpoint, CheckpointConfig, CheckpointId, CheckpointMetadata, Checkpointer, PendingWrite, - merge_writes, require_checkpoint_id, + decode_json_err, merge_writes, require_checkpoint_id, }; /// How many times a compare-and-swap loop retries before giving up. @@ -50,11 +50,6 @@ fn map_error(error: StorageError) -> TinyAgentsError { TinyAgentsError::Checkpoint(format!("storage driver: {error}")) } -/// Documents read per driver call when following a query to its end. -/// [`DocumentStoreExt::query_all`] follows the cursors, so this bounds one -/// page, not the result. -const PAGE: usize = 500; - /// Longest id stored as is; longer ones are hashed. const MAX_KEY_LEN: usize = 400; @@ -191,8 +186,7 @@ impl DriverCheckpointer { async fn thread_docs(&self, thread: &str) -> Result>> { self.declared().await?; let query = Query::filter(Filter::eq("thread", thread)) - .sort(Sort::asc("seq")) - .limit(PAGE); + .sort(Sort::asc("seq")); self.docs .query_all(&self.checkpoints, &query) .await @@ -218,10 +212,11 @@ where { fn decode(stored: Versioned) -> Result> { let record = stored.doc.get("record").cloned().unwrap_or(Value::Null); - let mut checkpoint: Checkpoint = - serde_json::from_value(record).map_err(|error| { - TinyAgentsError::Checkpoint(format!("storage driver: decode: {error}")) - })?; + // Through the shared classifier, so a `State` that no longer decodes + // is tagged `[schema]` exactly as the file and SQLite backends tag it + // (durable delegations prune such a checkpoint and start fresh). + let mut checkpoint: Checkpoint = serde_json::from_value(record) + .map_err(|error| decode_json_err("storage driver checkpointer", "record", error))?; checkpoint.normalize(); Ok(checkpoint) } @@ -315,7 +310,7 @@ where self.declared().await?; let counters = self .docs - .query_all(&self.threads, &Query::all().limit(PAGE)) + .query_all(&self.threads, &Query::all()) .await .map_err(map_error)?; let mut threads = Vec::new(); diff --git a/crates/tinyagents-harness/src/store/drivers.rs b/crates/tinyagents-harness/src/store/drivers.rs index c121d5dbc..252e5c3dc 100644 --- a/crates/tinyagents-harness/src/store/drivers.rs +++ b/crates/tinyagents-harness/src/store/drivers.rs @@ -13,9 +13,10 @@ //! (default [`DriverStore::DEFAULT_COLLECTION`]). Each entry is a document //! `{ "ns": , "key": , "value": }`, so a namespace //! or key may hold any characters and `list` is an indexed query on `ns`. -//! - [`DriverAppendStore`] maps each harness stream to a driver stream of the -//! same name, or with a prefix to `:`; the length -//! keeps two prefixes from ever addressing one stream. Offsets are the +//! - [`DriverAppendStore`] maps each harness stream to the driver stream +//! `:`, the prefix empty when there is none. The +//! length is always written, so no two `(prefix, stream)` pairs, prefixed +//! or not, ever address one stream. Offsets are the //! driver's dense offsets, which already match the [`AppendStore`] //! contract. @@ -130,7 +131,7 @@ impl Store for DriverStore { async fn list(&self, namespace: &str) -> Result> { self.declared().await?; - let query = Query::filter(Filter::eq("ns", namespace)).limit(1_000); + let query = Query::filter(Filter::eq("ns", namespace)); let found = self .docs .query_all(&self.collection, &query) @@ -157,7 +158,7 @@ pub struct DriverAppendStore { } impl DriverAppendStore { - /// Map each harness stream to the driver stream of the same name. + /// Map each harness stream `s` to the driver stream `0:s`. pub fn new(streams: Arc) -> Self { Self::with_prefix(streams, "") } @@ -173,11 +174,7 @@ impl DriverAppendStore { } fn name(&self, stream: &str) -> String { - if self.prefix.is_empty() { - stream.to_owned() - } else { - format!("{}:{}{stream}", self.prefix.len(), self.prefix) - } + format!("{}:{}{stream}", self.prefix.len(), self.prefix) } } diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index 29aea491f..baaa38888 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -50,9 +50,6 @@ pub(super) const INDEX: &str = "session_transcripts"; /// One document per write to a transcript: its append-only log. pub(super) const ENTRIES: &str = "session_transcript_entries"; -/// Entries read per driver round trip while replaying a log. -const PAGE: usize = 500; - /// Insert attempts before a contended write gives up. const CAS_ATTEMPTS: usize = 64; @@ -358,10 +355,7 @@ impl HistoryInner { let query = Query::filter( Filter::eq("stem", self.stem.as_str()).and(Filter::gte("seq", replay.next_seq)), ) - .sort(Sort::asc("seq")) - .limit(PAGE); - // `query_all` follows the cursors: `PAGE` bounds one round trip, not - // the replay. + .sort(Sort::asc("seq")); for stored in self.docs.query_all(ENTRIES, &query).await? { let seq = stored .doc @@ -469,9 +463,12 @@ impl HistoryInner { .doc .get("indexed_seq") .and_then(Value::as_u64) - .is_some_and(|indexed| indexed > seq) + .is_some_and(|indexed| indexed >= seq) }); if newer { + // Someone indexed this entry or a later one: the document is + // at least as fresh as this replay. + replay.indexed = Some(fields); return Ok(()); } let created_at = existing diff --git a/crates/tinyagents-session/src/port/drivers/turn_states.rs b/crates/tinyagents-session/src/port/drivers/turn_states.rs index 0afcd1ef4..22d87f7d2 100644 --- a/crates/tinyagents-session/src/port/drivers/turn_states.rs +++ b/crates/tinyagents-session/src/port/drivers/turn_states.rs @@ -139,7 +139,7 @@ impl Inner { async fn query(&self, filter: Filter) -> Result>, StorageError> { self.declared().await?; self.docs - .query_all(COLLECTION, &Query::filter(filter).limit(1_000)) + .query_all(COLLECTION, &Query::filter(filter)) .await? .into_iter() .map(decode) From 06c195a4244ef6c5878a416792088eda10619664 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:43:49 +0300 Subject: [PATCH 51/56] fix(session): repair transcript index on read A read now refreshes the index document when a previous write's index refresh failed, so transcripts that are never written again stay visible to thread and agent lookups. The repair is best effort and logs failures at debug level. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index baaa38888..907f89061 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -430,6 +430,21 @@ impl HistoryInner { ) } + /// Brings the index up to this replay on a read, best effort. A write + /// whose index refresh failed is otherwise only repaired by the next + /// write, and a transcript nobody writes again would stay invisible to + /// thread and agent lookups. Usually one read of a fresh index document. + async fn repair_index(&self, replay: &mut Replay) { + if let Err(error) = self.index(replay).await { + tracing::debug!( + target: "tinyagents_session::port::drivers", + stem = %self.stem, + %error, + "[session-store] transcript index repair on read failed" + ); + } + } + /// Refreshes this stem's [`INDEX`] document when its lookup fields /// changed. /// @@ -507,6 +522,7 @@ impl TranscriptRead for DriverTranscriptHistory { Ok(run_on(&self.bridge, async move { let mut replay = inner.replay.lock().await; inner.refresh(&mut replay).await?; + inner.repair_index(&mut replay).await; Ok(replay.written.then(|| SessionTranscript { meta: replay.meta.clone().unwrap_or_else(|| inner.seed.clone()), messages: replay.messages.clone(), From b65bbc3b342532a2b8e2fd7fe2715eb8bcb22152 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:44:03 +0300 Subject: [PATCH 52/56] fix(session): fail closed when in-flight turn recovery fails Recovery failures now propagate as an unavailable storage error instead of returning usable stores, so a retried sweep cannot mistake a new turn for crash residue. Destination keys and handle paths also use a per-provider UUID rather than the backend's address, which the allocator may reuse. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/mod.rs | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index 9b9d71347..564d888ab 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -63,6 +63,8 @@ pub struct DriverSessionStores { agents: Mutex>, recover_on_open: bool, recovered: Mutex>, + /// Names this provider in destination keys and handle paths. + id: uuid::Uuid, } impl std::fmt::Debug for DriverSessionStores { @@ -95,6 +97,7 @@ impl DriverSessionStores { agents: Mutex::new(HashMap::new()), recover_on_open: false, recovered: Mutex::new(HashSet::new()), + id: uuid::Uuid::new_v4(), } } @@ -140,9 +143,11 @@ impl DriverSessionStores { } let scoped = self.backend.for_scope(&Self::scope_for(agent_id))?; let stores = self.build(&scoped); - if self.recover_on_open && !self.recover_agent(agent_id, &stores) { - // Not cached, so the next open tries the recovery again. - return Ok(stores); + if self.recover_on_open { + // Fail closed: stores whose recovery did not run must not start + // turns a retried sweep would then mistake for crash residue. + // Nothing is cached, so the next open tries again. + self.recover_agent(agent_id, &stores)?; } Ok(self .agents @@ -154,19 +159,19 @@ impl DriverSessionStores { } /// Interrupts `agent_id`'s in-flight turns unless this provider already - /// did; returns whether the agent is now recovered. + /// did. /// /// The `recovered` lock is held across the sweep, so a concurrent first /// open of the same agent waits for it instead of handing out stores a /// still-running sweep could interrupt a new turn on. The agent is /// recorded only once the sweep succeeds. - fn recover_agent(&self, agent_id: &str, stores: &AgentStores) -> bool { + fn recover_agent(&self, agent_id: &str, stores: &AgentStores) -> Result<(), StorageError> { let mut recovered = self .recovered .lock() .unwrap_or_else(PoisonError::into_inner); if recovered.contains(agent_id) { - return true; + return Ok(()); } let now = chrono::Utc::now().to_rfc3339(); match stores.turn_states.mark_all_interrupted(&now) { @@ -180,7 +185,7 @@ impl DriverSessionStores { ); } recovered.insert(agent_id.to_string()); - true + Ok(()) } Err(error) => { tracing::warn!( @@ -189,21 +194,19 @@ impl DriverSessionStores { %error, "[session-store] could not recover in-flight turns; retried on the next open" ); - false + Err(StorageError::unavailable(format!( + "recovering in-flight turns failed: {error}" + ))) } } } fn build(&self, scoped: &ScopedStorage) -> AgentStores { let docs = Arc::clone(scoped.documents()); - // The backend's address keeps two backends with the same driver and - // scope from claiming one destination. - let label = format!( - "{}://{:p}/{}", - scoped.driver(), - Arc::as_ptr(&self.backend), - scoped.scope() - ); + // The provider's own id keeps two providers with the same driver and + // scope from claiming one destination; unlike an address, it is + // never reused. + let label = format!("{}://{}/{}", scoped.driver(), self.id, scoped.scope()); AgentStores { transcripts: Arc::new(DriverTranscriptLocator::new( Arc::clone(&docs), @@ -264,11 +267,7 @@ impl SessionStoreProvider for DriverSessionStores { } fn destination_key(&self) -> Option { - Some(format!( - "{}://{:p}", - self.backend.driver(), - Arc::as_ptr(&self.backend) - )) + Some(format!("{}://{}", self.backend.driver(), self.id)) } } From 424c5695f5de153755cbef3bc0abff3a193311a1 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:44:32 +0300 Subject: [PATCH 53/56] test(session): cover recovery failure and index repair paths Add tests for a flaky backend whose queries fail, verifying that a failed recovery sweep fails closed and is retried on the next call, and that a read repairs a session index a lost write refresh left stale. Also relax the bare stream name assertion in the harness store tests to match the encoding behaviour and document that recovery failures fail closed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/store/drivers_tests.rs | 8 +- .../src/port/drivers/mod.rs | 5 +- .../src/port/drivers/mod_tests.rs | 163 ++++++++++++++++++ .../src/port/drivers/transcripts_tests.rs | 18 ++ 4 files changed, 190 insertions(+), 4 deletions(-) diff --git a/crates/tinyagents-harness/src/store/drivers_tests.rs b/crates/tinyagents-harness/src/store/drivers_tests.rs index 8a6465db4..d044d4ce8 100644 --- a/crates/tinyagents-harness/src/store/drivers_tests.rs +++ b/crates/tinyagents-harness/src/store/drivers_tests.rs @@ -118,6 +118,10 @@ async fn read_from_follows_long_streams_and_prefixes_isolate() { ab.append("c", json!("ab/c")).await.unwrap(); assert_eq!(a.len("bc").await.unwrap(), 0, "prefixes never overlap"); let bare = DriverAppendStore::new(Arc::clone(scoped(&storage, "local").streams())); - let error = bare.append("", json!(1)).await.unwrap_err(); - assert!(matches!(error, TinyAgentsError::Validation(_)), "{error:?}"); + bare.append("", json!("bare")).await.unwrap(); + assert_eq!(bare.len("").await.unwrap(), 1, "a bare name is encoded too"); + // A bare stream spelled like a prefixed one stays its own stream. + bare.append("1:ab", json!("bare")).await.unwrap(); + assert_eq!(a.len("b").await.unwrap(), 0); + assert_eq!(a.len("").await.unwrap(), 0); } diff --git a/crates/tinyagents-session/src/port/drivers/mod.rs b/crates/tinyagents-session/src/port/drivers/mod.rs index 564d888ab..abcc75821 100644 --- a/crates/tinyagents-session/src/port/drivers/mod.rs +++ b/crates/tinyagents-session/src/port/drivers/mod.rs @@ -229,8 +229,9 @@ impl DriverSessionStores { impl SessionStoreProvider for DriverSessionStores { /// The stores of `agent_id`. /// - /// Fails closed: when the backend cannot bind the agent's scope, the - /// stores returned refuse every call with that error rather than fall + /// Fails closed: when the backend cannot bind the agent's scope, or + /// [`Self::recover_on_open`]'s sweep fails, the stores returned refuse + /// every call with that error rather than fall /// back to somewhere the agent's data does not belong. They are not /// cached, so the next call tries the backend again. fn for_agent(&self, agent_id: &str) -> AgentStores { diff --git a/crates/tinyagents-session/src/port/drivers/mod_tests.rs b/crates/tinyagents-session/src/port/drivers/mod_tests.rs index e4841b9e6..66ebeffd0 100644 --- a/crates/tinyagents-session/src/port/drivers/mod_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/mod_tests.rs @@ -267,3 +267,166 @@ async fn an_unbindable_scope_fails_closed() { ); provider.recover().unwrap(); } + +/// A memory backend whose documents fail every query while `down` is set. +#[derive(Debug)] +struct Flaky { + storage: MemoryStorage, + down: Arc, +} + +#[derive(Debug)] +struct FlakyDocs { + inner: Arc, + down: Arc, +} + +impl FlakyDocs { + fn check(&self) -> Result<(), StorageError> { + if self.down.load(std::sync::atomic::Ordering::SeqCst) { + Err(StorageError::unavailable("flaky backend is down")) + } else { + Ok(()) + } + } +} + +#[tinystoragedrivers_core::async_trait] +impl tinystoragedrivers_core::DocumentStore for FlakyDocs { + fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + self.inner.capabilities() + } + async fn ensure_collection( + &self, + spec: &tinystoragedrivers_core::CollectionSpec, + ) -> Result<(), StorageError> { + self.inner.ensure_collection(spec).await + } + async fn get( + &self, + collection: &str, + id: &str, + ) -> Result>, StorageError> { + self.inner.get(collection, id).await + } + async fn put( + &self, + collection: &str, + id: &str, + doc: serde_json::Value, + precondition: tinystoragedrivers_core::Precondition, + ) -> Result { + self.inner.put(collection, id, doc, precondition).await + } + async fn delete( + &self, + collection: &str, + id: &str, + precondition: tinystoragedrivers_core::Precondition, + ) -> Result { + self.inner.delete(collection, id, precondition).await + } + async fn query( + &self, + collection: &str, + query: &tinystoragedrivers_core::Query, + ) -> Result< + tinystoragedrivers_core::Page>, + StorageError, + > { + self.check()?; + self.inner.query(collection, query).await + } + async fn count( + &self, + collection: &str, + filter: &tinystoragedrivers_core::Filter, + ) -> Result { + self.inner.count(collection, filter).await + } + async fn delete_where( + &self, + collection: &str, + filter: &tinystoragedrivers_core::Filter, + ) -> Result { + self.inner.delete_where(collection, filter).await + } + async fn claim( + &self, + collection: &str, + filter: &tinystoragedrivers_core::Filter, + sort: &[tinystoragedrivers_core::Sort], + patch: &serde_json::Value, + ) -> Result>, StorageError> { + self.inner.claim(collection, filter, sort, patch).await + } + async fn drop_collection(&self, collection: &str) -> Result<(), StorageError> { + self.inner.drop_collection(collection).await + } +} + +impl StorageBackend for Flaky { + fn driver(&self) -> &'static str { + "flaky" + } + fn capabilities(&self) -> tinystoragedrivers_core::Capabilities { + self.storage.capabilities() + } + fn for_scope(&self, scope: &Scope) -> Result { + let inner = self.storage.for_scope(scope)?; + Ok(ScopedStorage::new( + scope.clone(), + "flaky", + Arc::new(FlakyDocs { + inner: Arc::clone(inner.documents()), + down: Arc::clone(&self.down), + }), + Arc::clone(inner.streams()), + Arc::clone(inner.blobs()), + )) + } + fn database(&self, name: &str) -> Result, StorageError> { + self.storage.database(name) + } +} + +#[test] +fn a_failed_recovery_fails_closed_and_is_retried() { + let down = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let provider = DriverSessionStores::new(Arc::new(Flaky { + storage: MemoryStorage::new(), + down: Arc::clone(&down), + })) + .unwrap() + .recover_on_open(true); + + let error = provider.try_for_agent("a").unwrap_err(); + assert!(error.message().contains("recovering"), "{error}"); + let refused = provider.for_agent("a"); + assert!( + refused + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .is_err(), + "no turn can start before recovery ran" + ); + assert!(provider.agents.lock().unwrap().is_empty()); + + down.store(false, std::sync::atomic::Ordering::SeqCst); + let stores = provider.for_agent("a"); + stores + .turn_states + .put(&TurnState::started("t", "r", 8, "2026-01-01T00:00:00Z")) + .unwrap(); + assert_eq!( + provider + .for_agent("a") + .turn_states + .get("t") + .unwrap() + .unwrap() + .lifecycle, + TurnLifecycle::Started, + "a recovered agent is never swept again" + ); +} diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 485f14f2b..e876555ee 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -590,3 +590,21 @@ fn a_stale_handle_never_rewinds_the_index() { assert!(locator.root_for_thread("new").is_some()); assert!(locator.root_for_thread("old").is_none()); } + +#[test] +fn a_read_repairs_an_index_a_write_could_not_refresh() { + let docs = docs(); + let locator = locator(&docs); + let history = locator.handle("s", meta("t")); + history.append(message("user", "a")).unwrap(); + // The index refresh after that write was lost. + let raw = Arc::clone(&docs); + on_bridge(async move { raw.delete(INDEX, &doc_key(&["s"]), Precondition::None).await }); + assert!(locator.root_for_thread("t").is_none()); + locator + .handle("s", meta("t")) + .read_session() + .unwrap() + .unwrap(); + assert!(locator.root_for_thread("t").is_some(), "the read re-indexed it"); +} From 6cf3b98844d616ff9b61188c0448c3fb579f1a87 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:45:32 +0300 Subject: [PATCH 54/56] style: reformat long expressions to satisfy line width Reformatted a query builder chain in the checkpoint driver and wrapped several long expressions in the transcript tests so they fit within the line width limit. No behaviour changed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinyagents-graph/src/checkpoint/drivers.rs | 3 +-- .../src/port/drivers/transcripts_tests.rs | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/tinyagents-graph/src/checkpoint/drivers.rs b/crates/tinyagents-graph/src/checkpoint/drivers.rs index 7cbad5acb..cc4f24310 100644 --- a/crates/tinyagents-graph/src/checkpoint/drivers.rs +++ b/crates/tinyagents-graph/src/checkpoint/drivers.rs @@ -185,8 +185,7 @@ impl DriverCheckpointer { /// Every stored checkpoint document of `thread`, in insertion order. async fn thread_docs(&self, thread: &str) -> Result>> { self.declared().await?; - let query = Query::filter(Filter::eq("thread", thread)) - .sort(Sort::asc("seq")); + let query = Query::filter(Filter::eq("thread", thread)).sort(Sort::asc("seq")); self.docs .query_all(&self.checkpoints, &query) .await diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index e876555ee..1f3893d65 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -599,12 +599,18 @@ fn a_read_repairs_an_index_a_write_could_not_refresh() { history.append(message("user", "a")).unwrap(); // The index refresh after that write was lost. let raw = Arc::clone(&docs); - on_bridge(async move { raw.delete(INDEX, &doc_key(&["s"]), Precondition::None).await }); + on_bridge(async move { + raw.delete(INDEX, &doc_key(&["s"]), Precondition::None) + .await + }); assert!(locator.root_for_thread("t").is_none()); locator .handle("s", meta("t")) .read_session() .unwrap() .unwrap(); - assert!(locator.root_for_thread("t").is_some(), "the read re-indexed it"); + assert!( + locator.root_for_thread("t").is_some(), + "the read re-indexed it" + ); } From bc7511240432ead47756cda833bade80b7970102 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:47:10 +0300 Subject: [PATCH 55/56] docs(session): clarify journal stream prefix description Reworded the journal row in the store port documentation to state that the harness `DriverAppendStore` always writes the prefix length first, replacing the vaguer note about length-prefixing its prefix. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/modules/session/store-port.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/session/store-port.md b/docs/modules/session/store-port.md index 481c4774e..f9d9af91c 100644 --- a/docs/modules/session/store-port.md +++ b/docs/modules/session/store-port.md @@ -83,7 +83,7 @@ provider claims isolation and passes `session_store_isolation_conformance`. | transcripts | `session_transcripts`: one index document per stem (thread, agent, sub-agent flag, creation time); `session_transcript_entries`: an append-only log per stem | | turn states | `session_turn_states`: one document per `(thread, request)` | | key-value | `session_kv` through the harness `DriverStore` | -| journal | streams `16:session_journal/` (the harness `DriverAppendStore` length-prefixes its prefix) | +| journal | streams `16:session_journal/` (the harness `DriverAppendStore` always writes `:` first) | - **Transcript log.** Each write is one entry, numbered from 0 and claimed with an insert-only write. An ordinary turn stores only its new rows From 0186a3f423be5b14cfd7866e5a04f072eab3d749 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 9 Oct 2026 05:54:55 +0300 Subject: [PATCH 56/56] feat(session): guard transcript generations with claimed reservations A handle opened for a new generation now holds its reservation until its first write claims it via compare-and-swap, so a stalled owner whose reservation was taken over has its writes refused instead of overwriting the new generation. Clearing an unwritten successor releases the reservation immediately, and failed index refreshes are retried a few times before falling back to the next write or read. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/port/drivers/transcripts.rs | 119 +++++++++++++++++- .../src/port/drivers/transcripts_tests.rs | 53 ++++++++ 2 files changed, 166 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-session/src/port/drivers/transcripts.rs b/crates/tinyagents-session/src/port/drivers/transcripts.rs index 907f89061..fd0cc1855 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts.rs @@ -50,6 +50,10 @@ pub(super) const INDEX: &str = "session_transcripts"; /// One document per write to a transcript: its append-only log. pub(super) const ENTRIES: &str = "session_transcript_entries"; +/// Index refresh attempts after a durable write before leaving the repair +/// to the next write or read. +const INDEX_ATTEMPTS: usize = 3; + /// Insert attempts before a contended write gives up. const CAS_ATTEMPTS: usize = 64; @@ -165,6 +169,10 @@ struct Replay { sealed: bool, /// The lookup fields last written to [`INDEX`] by this handle. indexed: Option, + /// The version of the generation reservation this handle was opened + /// with, until its first write claims it (see + /// [`HistoryInner::claim_reservation`]). + reservation: Option, } impl Replay { @@ -388,6 +396,7 @@ impl HistoryInner { let Some(entry) = build(&replay, &self.seed)? else { return Ok(false); }; + self.claim_reservation(&mut replay).await?; let seq = replay.next_seq; let mut doc = serde_json::to_value(&entry).map_err(|error| serialization(&error))?; if let Value::Object(fields) = &mut doc { @@ -407,15 +416,26 @@ impl HistoryInner { Ok(_) => { replay.apply(seq, entry); // The entry is durable: report the write as done. A - // failed index refresh only delays lookups by thread or - // agent until this handle's next write retries it. - if let Err(error) = self.index(&mut replay).await { + // failed index refresh is retried a few times here, then + // by this handle's next write and by any read of the + // transcript; it never makes the durable write fail. + let mut last = None; + for _ in 0..INDEX_ATTEMPTS { + match self.index(&mut replay).await { + Ok(()) => { + last = None; + break; + } + Err(error) => last = Some(error), + } + } + if let Some(error) = last { tracing::warn!( target: "tinyagents_session::port::drivers", stem = %self.stem, seq, %error, - "[session-store] transcript index refresh failed; retried on the next write" + "[session-store] transcript index refresh failed; retried on the next write or read" ); } return Ok(true); @@ -430,6 +450,63 @@ impl HistoryInner { ) } + /// Claims the generation reservation this handle was opened with, once, + /// before its first write: a compare-and-swap from the reserved version. + /// If another process took the reservation over in the meantime (this + /// one stalled past [`STALE_RESERVATION_MS`]), the generation is theirs + /// and this handle's writes are refused instead of overwriting it. + async fn claim_reservation(&self, replay: &mut Replay) -> anyhow::Result<()> { + let Some(reserved) = replay.reservation else { + return Ok(()); + }; + let claim = json!({ + "stem": self.stem, + "subagent": is_subagent(&self.stem), + "written": false, + "created_at": now_rfc3339(), + "reserved_at": chrono::Utc::now().timestamp_millis(), + }); + match self + .docs + .put( + INDEX, + &doc_key(&[&self.stem]), + claim, + Precondition::Version(reserved), + ) + .await + { + Ok(_) => { + replay.reservation = None; + Ok(()) + } + Err(error) if error.kind() == ErrorKind::Conflict => anyhow::bail!( + "session generation {} was taken over by another writer; reload the session", + self.stem + ), + Err(error) => Err(error.into()), + } + } + + /// Releases the reservation of a generation this handle never wrote. + async fn release_reservation(&self, replay: &mut Replay) -> Result<(), StorageError> { + let Some(reserved) = replay.reservation.take() else { + return Ok(()); + }; + match self + .docs + .delete( + INDEX, + &doc_key(&[&self.stem]), + Precondition::Version(reserved), + ) + .await + { + Err(error) if error.kind() != ErrorKind::Conflict => Err(error), + _ => Ok(()), + } + } + /// Brings the index up to this replay on a read, best effort. A write /// whose index refresh failed is otherwise only repaired by the next /// write, and a transcript nobody writes again would stay invisible to @@ -659,6 +736,22 @@ impl TranscriptHistory for DriverTranscriptHistory { } fn clear(&self) -> anyhow::Result<()> { + // Clearing a reserved generation nobody wrote gives the reservation + // back, so the compaction can be retried at once instead of after + // the stale timeout. + let inner = Arc::clone(&self.inner); + let released = run_on(&self.bridge, async move { + let mut replay = inner.replay.lock().await; + inner.refresh(&mut replay).await?; + if replay.written || replay.reservation.is_none() { + return Ok(false); + } + inner.release_reservation(&mut replay).await?; + Ok(true) + })?; + if released { + return Ok(()); + } self.commit(|replay, _| { anyhow::ensure!(!replay.sealed, "transcript generation is sealed"); Ok(replay.written.then(|| Entry { @@ -708,13 +801,27 @@ impl DriverTranscriptLocator { } fn handle(&self, stem: &str, seed: TranscriptMeta) -> DriverTranscriptHistory { + self.reserved_handle(stem, seed, None) + } + + /// A handle that holds the generation reservation `reservation`; its + /// first write claims it (or is refused if it was taken over). + fn reserved_handle( + &self, + stem: &str, + seed: TranscriptMeta, + reservation: Option, + ) -> DriverTranscriptHistory { DriverTranscriptHistory { inner: Arc::new(HistoryInner { docs: Arc::clone(&self.docs), declared: Arc::clone(&self.declared), stem: stem.to_string(), seed, - replay: Mutex::new(Replay::default()), + replay: Mutex::new(Replay { + reservation, + ..Replay::default() + }), }), bridge: self.bridge.clone(), path: PathBuf::from(format!("{}/{stem}", self.label)), @@ -995,7 +1102,7 @@ impl DriverTranscriptLocator { let mut meta = seed; meta.session_id = Some(successor.session_id()); meta.parent_session_id = successor.parent_session_id(); - let handle = self.handle(&stem, meta); + let handle = self.reserved_handle(&stem, meta, Some(reservation)); Ok((successor, Arc::new(handle))) } diff --git a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs index 1f3893d65..69e762cfd 100644 --- a/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs +++ b/crates/tinyagents-session/src/port/drivers/transcripts_tests.rs @@ -614,3 +614,56 @@ fn a_read_repairs_an_index_a_write_could_not_refresh() { "the read re-indexed it" ); } + +#[test] +fn clearing_an_unwritten_successor_frees_the_generation() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + locator + .open_session(&session, meta("t")) + .unwrap() + .append(message("user", "a")) + .unwrap(); + let (_, next) = locator.begin_generation(&session, meta("t")).unwrap(); + next.clear().unwrap(); + let (successor, again) = locator + .begin_generation(&session, meta("t")) + .expect("the cleared reservation is free at once"); + again.append(message("user", "summary")).unwrap(); + assert_eq!(locator.head_generation(&session), successor); +} + +#[test] +fn a_taken_over_generation_refuses_its_stalled_owner() { + let docs = docs(); + let locator = locator(&docs); + let session = SessionRef::scoped("t", "planner"); + locator + .open_session(&session, meta("t")) + .unwrap() + .append(message("user", "a")) + .unwrap(); + let (_, stalled) = locator.begin_generation(&session, meta("t")).unwrap(); + // The owner stalls past the stale timeout and another process takes over. + let id = doc_key(&[&session_stem(&session.next_generation())]); + let aged = Arc::clone(&docs); + on_bridge(async move { + let found = aged.get(INDEX, &id).await?.unwrap(); + let mut doc = found.doc; + doc["reserved_at"] = json!(0); + aged.put(INDEX, &id, doc, Precondition::None).await + }); + let (_, winner) = locator.begin_generation(&session, meta("t")).unwrap(); + winner.append(message("user", "winner")).unwrap(); + + let error = stalled.replace(&[message("user", "late")]).unwrap_err(); + assert!(error.to_string().contains("taken over"), "{error}"); + let contents: Vec = winner + .messages() + .unwrap() + .into_iter() + .map(|row| row.content) + .collect(); + assert_eq!(contents, ["winner"]); +}