diff --git a/changelog.d/timeline-otel-cursor.fixed.md b/changelog.d/timeline-otel-cursor.fixed.md new file mode 100644 index 000000000..b64dc5fb3 --- /dev/null +++ b/changelog.d/timeline-otel-cursor.fixed.md @@ -0,0 +1 @@ +OTLP export no longer drops a span whose record shares the millisecond the previous batch ended on (a tool call that ends at the same millisecond it started, or a turn ending at the last record's time): the export cursor re-reads records at its last timestamp and deduplicates spans by id. diff --git a/docs/subsystems/timeline.md b/docs/subsystems/timeline.md index cf0658c47..377d73340 100644 --- a/docs/subsystems/timeline.md +++ b/docs/subsystems/timeline.md @@ -51,7 +51,7 @@ Everything in `plugins/timeline/`: the recorder, the trace file, the read model 8. The panel draws text only: every string is a React text child or an SVG text node, never markup, and is cleaned again in the browser (`safe`). It polls only while open (and refreshes at once when the tab or window comes back, #259, #271), shows the last good data under an error, and says in a note when the trace is shorter than the window, truncated, memory-only or looking back. Its one-line footnote repeats the honesty rules of invariant 5 (delivered is not read, busy is not idle, model time is not a measured latency). The dialog shrinks to its content and hangs from the top (switching tabs moves nothing above the body), the **Live** control is always shown (pressed while following the latest activity, a call to action when looking back), lanes name what they are under the name (`main`, `group`, `friend`, `removed`) with no icon prefix, and the map puts each name beside or under its node, clipped with the full name in a tooltip (#249). A turn bar's tooltip ends with the model the turn ran on (its model id, `· deepseek-v4-pro`); a running turn, or one recorded before #366, names none (#366). A friend's Bot is named as the sidebar names it (`UiA1415's Chief`, `Helper · UiB1415's Bot`, from the feed's `remote.name` / party `name`), so two Bots called Helper stay apart; a wait label that would run past the plot is right-aligned, and one with no room is left to the arrow's tooltip (#255). The window's range is always in view under the header (`Oct 18 · 13:20:05 – 13:35:05 (15 min)`, dates at both ends across midnight, and whether it is live), and the footnote is two short items, not one line (#279). 9. Opt-in (#281): `apply` without `config.enabled === true` subscribes to nothing, writes nothing and registers no endpoint; the browser entry draws no sidebar row unless `config.sidebarSection === true` and its poller only runs while a panel is open. A test asserts the host does nothing by default. Docs and the install doctor's default commands do not present it as part of a standard install. 10. The endpoint reads only. It does not start, stop, send or change anything. -11. OTLP export (#483) is opt-in and off by default: without `config.exporter` nothing leaves the host. With `exporter: "otlp"` the plugin POSTs OTLP/HTTP JSON to `otlpUrl`; with `"json"` it appends `dir/export.jsonl` (capped at 8 MiB, then it stops and warns). `otlpUrl` must be `https`, or `http` on localhost only (the request may carry an auth header). Only CLOSED spans are exported: a turn needs its `te` and a tool call its `ke`, so an open turn is carried between flushes and exported when it ends, never as a finished success. The exporter reads only the same compact records the panel does (invariant 2 still holds: no message text, arguments or output), maps turns to `invoke_agent` spans and tool calls to `execute_tool` spans under them, and carries every id through the same deterministic hash. Bot and room NAMES are sent only with `exportNames: true`; by default a span carries ids only, so a friend's or another home's name never leaves the host (a name comes from `trace.names()`, which includes friends' Bots and face-room names). The feed has no model name and no token counts, so those GenAI attributes are absent, not guessed. A failed send keeps the cursor, so the next flush retries the same closed spans (tested). +11. OTLP export (#483) is opt-in and off by default: without `config.exporter` nothing leaves the host. With `exporter: "otlp"` the plugin POSTs OTLP/HTTP JSON to `otlpUrl`; with `"json"` it appends `dir/export.jsonl` (capped at 8 MiB, then it stops and warns). `otlpUrl` must be `https`, or `http` on localhost only (the request may carry an auth header). Only CLOSED spans are exported: a turn needs its `te` and a tool call its `ke`, so an open turn is carried between flushes and exported when it ends, never as a finished success. The exporter reads only the same compact records the panel does (invariant 2 still holds: no message text, arguments or output), maps turns to `invoke_agent` spans and tool calls to `execute_tool` spans under them, and carries every id through the same deterministic hash. Bot and room NAMES are sent only with `exportNames: true`; by default a span carries ids only, so a friend's or another home's name never leaves the host (a name comes from `trace.names()`, which includes friends' Bots and face-room names). The feed has no model name and no token counts, so those GenAI attributes are absent, not guessed. A failed send keeps the cursor, so the next flush retries the same closed spans (tested). The cursor re-reads records at its last timestamp (`>=`), because a `ke` can share the millisecond of its `ks` (or a `te` the time of the last record); a span already sent is deduplicated by id, so a same-millisecond record is neither skipped nor sent twice. ## Tests diff --git a/plugins/timeline/src/otel.js b/plugins/timeline/src/otel.js index 63e0e513b..81485c7ca 100644 --- a/plugins/timeline/src/otel.js +++ b/plugins/timeline/src/otel.js @@ -173,10 +173,21 @@ export function createExporter({ send, serviceName = 'flybot', serviceVersion = } return { async push(records, nameTables = {}) { - const fresh = records.filter(record => record && Number.isFinite(record.t) && record.t > lastT) + // `>=`, not `>`: a record can share the millisecond the previous batch ended on (a tool's + // `ke` landing at the same ms as its `ks`, or a `te` at the last record's time). With `>` it + // would never be picked up again — it is not an open record to carry — and its span was lost. + // Records at `lastT` are re-read each push; `emitted` keeps their spans from being sent twice. + const fresh = records.filter(record => record && Number.isFinite(record.t) && record.t >= lastT) if (fresh.length < minRecords) return 0 const batch = [...carried, ...fresh] - const spans = spansFrom(batch, { names: exportNames ? nameTables : {} }).filter(span => !emitted.has(span.spanId)) + // A record at `lastT` may already be in `carried`, so a batch can repeat it; one span per id. + const spans = [] + const seen = new Set() + for (const span of spansFrom(batch, { names: exportNames ? nameTables : {} })) { + if (emitted.has(span.spanId) || seen.has(span.spanId)) continue + seen.add(span.spanId) + spans.push(span) + } const nextCarried = pendingRecords(batch) const nextT = Math.max(lastT, ...fresh.map(record => record.t)) if (spans.length === 0) { carried = nextCarried; lastT = nextT; return 0 } diff --git a/plugins/timeline/test/otel.test.js b/plugins/timeline/test/otel.test.js index 8f5a76235..ea2643585 100644 --- a/plugins/timeline/test/otel.test.js +++ b/plugins/timeline/test/otel.test.js @@ -69,6 +69,20 @@ test('createExporter holds an open turn, sends it once it closes, and never twic assert.deepEqual(spanNames.sort(), ['execute_tool web_search', 'invoke_agent']) }) +test('createExporter does not lose a record that shares the timestamp of the previous batch', async () => { + const sent = [] + const exporter = createExporter({ send: async (payload, info) => sent.push({ payload, info }) }) + const first = [{ r: 'ts', t: 1_000, id: 'a#1', pl: 'chat', bot: 'a' }, { r: 'ks', t: 1_100, id: 'c1', tid: 'a#1', nm: 'web_search', bot: 'a' }] + assert.equal(await exporter.push(first), 0, 'nothing closed yet; the batch ends on the 1_100 ms tool start') + // The tool ends at the same 1_100 ms the previous batch ended on; it must still be exported. + const withKe = [...first, { r: 'ke', t: 1_100, id: 'c1', tid: 'a#1', ok: true }] + assert.equal(await exporter.push(withKe), 1, 'the tool span is exported') + const withTe = [...withKe, { r: 'te', t: 1_200, id: 'a#1', o: 'completed' }] + assert.equal(await exporter.push(withTe), 1, 'the turn span is exported') + const spanNames = sent.flatMap(item => item.payload.resourceSpans[0].scopeSpans[0].spans.map(span => span.name)) + assert.deepEqual(spanNames.sort(), ['execute_tool web_search', 'invoke_agent']) +}) + test('createExporter retries the same closed spans after a failed send', async () => { let fail = true const exporter = createExporter({ send: async () => { if (fail) throw new Error('nope') }, warn: () => {} })