From 4cd0e4e17340530f0a67b66c9f901ddad589fc02 Mon Sep 17 00:00:00 2001 From: Richard Palethorpe Date: Fri, 12 Jun 2026 11:07:19 +0100 Subject: [PATCH] feat(capi): ABI v5 - surface vs distinction across the C boundary The streaming C API conflated end-of-utterance and backchannel: the feed eou_out flag and the JSON "eou" field were 1 for either token, so a voice agent could not tell "the user yielded the turn" (, respond) from "the user acknowledged while listening" (, do not treat as a turn). - parakeet_capi_stream_feed's *eou_out is now an event bitmask (PARAKEET_EVENT_EOU | PARAKEET_EVENT_EOB) instead of an any-event 0/1 - the streaming JSON documents carry separate "eou"/"eob" 0/1 flags plus an "events" array ({"type":"eou"|"eob","frame","t"}) with per-event timestamps - new parakeet_capi_stream_drain_events / parakeet_capi_free_events expose typed per-event records (token, is_eob, encoder_frame, time_sec) on the flat API; the queue is shared with the JSON entry points - StreamingSession gains a non-draining events() peek; the C layer watermarks the queue to attribute events to one feed pass (a feed can span several encoder chunks, which last_chunk_had_eou() cannot express) - decoder reset-on-/ is deliberately unchanged (matches NeMo's reference driver); the distinction is a downstream turn-taking signal - tests: both C-API stream tests gain a two-utterance-clip phase (shared builder in tests/stream_clips.hpp, same construction as gen_stream_reset_baseline.py) asserting the typed mid-stream and exact flag/mask agreement with the drained queue Verified against mudler/parakeet-cpp-gguf realtime_eou_120m-v1-f16: all 61 ctest targets pass; the two-utterance clip yields the mid-stream at t=7.84s with mask=PARAKEET_EVENT_EOU. Assisted-by: Claude:claude-fable-5 [Claude Code] --- AGENTS.md | 12 +++- docs/parity.md | 11 ++- include/parakeet_capi.h | 69 +++++++++++++++--- src/parakeet_capi.cpp | 119 ++++++++++++++++++++++++++------ src/streaming.hpp | 5 ++ tests/stream_clips.hpp | 17 +++++ tests/test_capi_stream.cpp | 108 +++++++++++++++++++++++++++-- tests/test_capi_stream_json.cpp | 72 ++++++++++++++++++- 8 files changed, 373 insertions(+), 40 deletions(-) create mode 100644 tests/stream_clips.hpp diff --git a/AGENTS.md b/AGENTS.md index d36fe45..f512a53 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -253,7 +253,7 @@ parakeet_capi_free_string parakeet_capi_last_error # streaming (cache-aware EOU model parakeet_realtime_eou_120m-v1): parakeet_capi_stream_begin -parakeet_capi_stream_feed # 16k mono f32 PCM -> newly-finalized text; *eou_out=1 on / +parakeet_capi_stream_feed # 16k mono f32 PCM -> newly-finalized text; *eou_out = event bitmask (ABI v5) parakeet_capi_stream_finalize # flush the end-of-stream tail parakeet_capi_stream_free ``` @@ -274,6 +274,16 @@ semantics. Additive changes (new functions) are fine without bumping. Streaming semantics: `parakeet_capi_stream_feed` buffers PCM, decodes encoder chunks as audio arrives (carried encoder/decoder caches), and returns the newly-finalized text (``/`` STRIPPED, surfaced via `*eou_out`). +Since ABI v5 `*eou_out` is a bitmask — `PARAKEET_EVENT_EOU` (end of utterance: +respond) | `PARAKEET_EVENT_EOB` (backchannel: do not treat as a turn) — and the +streaming JSON documents carry separate `"eou"`/`"eob"` 0/1 flags (in v4 a +single conflated any-event flag). Per-event timestamps come from +`parakeet_capi_stream_drain_events` +(`parakeet_stream_event{token,is_eob,encoder_frame,time_sec}`, free with +`parakeet_capi_free_events`) or the `"events"` array +(`{"type":"eou"|"eob","frame","t"}`) in the `stream_feed_json` / +`stream_finalize_json` documents. The event queue is shared between the typed +drain and the JSON entry points — use one style per stream. `parakeet_capi_stream_finalize` flushes the streaming tail and does NOT fabricate an `` NeMo's cache-aware streaming would not emit (for a final chunk whose right context is incomplete, the trailing `` is dropped exactly diff --git a/docs/parity.md b/docs/parity.md index 159f582..2c00643 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -570,8 +570,17 @@ from `libparakeet.so` (verified via `nm -D`). - `tests/test_capi_stream.cpp` (`test_capi_stream`) — feeds `speech.wav` PCM in chunks through the streaming C-API; the concatenated text + `finalize` equals `baseline.stream_text` from `/tmp/baseline_eou_stream.gguf` (NeMo streaming). - Skips (exit 77) unless `PARAKEET_TEST_GGUF_EOU` + + A second phase streams a two-utterance clip (speech + 0.6 s silence + speech, + the `gen_stream_reset_baseline.py` construction) and asserts + `parakeet_capi_stream_drain_events` (ABI v5) surfaces the mid-stream `` + as a typed record agreeing with the per-feed `eou_out` event mask + (PARAKEET_EVENT_EOU | PARAKEET_EVENT_EOB), with sane monotone timestamps. Skips (exit 77) unless `PARAKEET_TEST_GGUF_EOU` + `PARAKEET_TEST_BASELINE_EOU_STREAM` are set. +- `tests/test_capi_stream_json.cpp` (`test_capi_stream_json`) — drives the + streaming JSON entry points on `speech.wav` and asserts the documents carry + `frame_sec`, per-word timestamps, and the `"events"` array; a second phase + streams the two-utterance clip and asserts a typed `{"type":"eou",...}` event + appears. Skips (exit 77) unless `PARAKEET_TEST_GGUF_EOU` is set. Reproduce: diff --git a/include/parakeet_capi.h b/include/parakeet_capi.h index b082455..c199a70 100644 --- a/include/parakeet_capi.h +++ b/include/parakeet_capi.h @@ -30,6 +30,16 @@ typedef struct parakeet_ctx parakeet_ctx; // (start/end/conf) plus frame_sec alongside the newly-finalized text, and // added "frame_sec" to the transcribe_*_json documents. The original entry // points are unchanged. +// +// v5: the (end of utterance) vs (end of backchannel) distinction is +// now visible across the C boundary. BREAKING semantics on the streaming +// surface: parakeet_capi_stream_feed's `*eou_out` is now a bitmask +// (PARAKEET_EVENT_EOU | PARAKEET_EVENT_EOB) instead of an any-event 0/1, +// and the JSON "eou" field now means "an fired" only, with a new +// "eob" field beside it (in v4 both meant "an OR fired"). +// Added parakeet_capi_stream_drain_events (typed per-event records with +// is_eob + timestamps, freed with parakeet_capi_free_events) and an +// "events" array in the stream_feed_json / stream_finalize_json documents. int parakeet_capi_abi_version(void); // Load a GGUF model. Returns an owning context, or NULL on failure. @@ -174,13 +184,22 @@ parakeet_stream* parakeet_capi_stream_begin(parakeet_ctx* ctx); parakeet_stream* parakeet_capi_stream_begin_lang(parakeet_ctx* ctx, const char* target_lang); +// Bits for parakeet_capi_stream_feed's *eou_out mask. = the user +// finished a complete utterance (a voice agent responds); = the user +// finished a backchannel, a short acknowledgment like "uh-huh" while the other +// party speaks (a voice agent must NOT treat it as the user taking the turn). +#define PARAKEET_EVENT_EOU 1 +#define PARAKEET_EVENT_EOB 2 + // Feed a block of 16 kHz MONO float PCM (`pcm`, length `n_samples`). The session // buffers the audio and decodes as full encoder chunks become available. // Returns the newly-finalized text since the last call as a malloc'd UTF-8 // string (free with parakeet_capi_free_string) — "" (empty, non-NULL) if no new // text was finalized this call, NULL only on error. / are stripped -// from the text and surfaced as events: if `eou_out` is non-NULL it is set to 1 -// when an / event fired during this feed, else 0. +// from the text and surfaced as events: if `eou_out` is non-NULL it is set to +// the bitwise OR of PARAKEET_EVENT_EOU / PARAKEET_EVENT_EOB for the event types +// that fired during this feed (0 if none). Per-event timestamps are available +// via parakeet_capi_stream_drain_events. char* parakeet_capi_stream_feed(parakeet_stream* s, const float* pcm, int n_samples, int* eou_out); @@ -190,16 +209,50 @@ char* parakeet_capi_stream_feed(parakeet_stream* s, const float* pcm, // complete. Does NOT fabricate an NeMo's streaming would not emit. char* parakeet_capi_stream_finalize(parakeet_stream* s); +// One / event emitted by the streaming decoder. marks the end +// of a complete utterance (the user yielded the turn); marks the end of a +// backchannel (a short acknowledgment like "uh-huh" while the other party +// speaks — a voice agent typically responds on but must NOT treat +// as the user taking the turn). time_sec is the absolute (stream-relative) +// emission time: encoder_frame * frame_sec. +typedef struct parakeet_stream_event { + int token; // raw vocab id of the special token + int is_eob; // 0 = (end of utterance), 1 = (backchannel) + int encoder_frame; // absolute encoder-output frame index of the emission + float time_sec; // encoder_frame * frame_sec, seconds from stream start +} parakeet_stream_event; + +// Drain the / events accumulated since the last drain. On success +// returns the event count (>= 0) and, when the count is nonzero, sets +// `*out_events` to a malloc'd array of that many records (release with +// parakeet_capi_free_events); `*out_events` is NULL when the count is 0. +// Returns -1 on error (NULL stream/out pointer) with `*out_events` NULL. +// The queue is shared with the JSON entry points: stream_feed_json / +// stream_finalize_json also drain it (into their "events" array), so use one +// style or the other per stream. +int parakeet_capi_stream_drain_events(parakeet_stream* s, + parakeet_stream_event** out_events); + +// Free an event array previously returned by parakeet_capi_stream_drain_events. +// Safe on NULL. +void parakeet_capi_free_events(parakeet_stream_event* events); + // Like parakeet_capi_stream_feed but returns a malloc'd UTF-8 JSON document // instead of bare text: -// {"text":"...","eou":0,"frame_sec":0.080000, +// {"text":"...","eou":0,"eob":0,"frame_sec":0.080000, +// "events":[{"type":"eou","frame":31,"t":2.480}, ...], // "words":[{"w":"...","start":0.480,"end":0.640,"conf":0.9100}, ...]} // "text" is the newly-finalized text since the last call ("" if none); "eou" is -// 1 iff an / fired during this feed; "frame_sec" is the encoder frame -// stride in seconds; "words" are the words finalized this call with absolute -// (stream-relative) start/end seconds and 'min'-aggregate confidence (the same -// drain as the offline pk::group_words). Returns NULL only on error (see -// parakeet_capi_last_error). Free with parakeet_capi_free_string. +// 1 iff an fired during this feed and "eob" 1 iff an fired (see +// parakeet_stream_event for the semantics — they are distinct turn-taking +// signals, not conflated); "frame_sec" is the encoder frame stride in seconds; +// "events" are the / events drained this call, each with "type" +// ("eou" = end of utterance, "eob" = backchannel), the absolute encoder frame +// and the emission time in seconds (frame * frame_sec); "words" are the words +// finalized this call with absolute (stream-relative) start/end seconds and +// 'min'-aggregate confidence (the same drain as the offline pk::group_words). +// Returns NULL only on error (see parakeet_capi_last_error). Free with +// parakeet_capi_free_string. char* parakeet_capi_stream_feed_json(parakeet_stream* s, const float* pcm, int n_samples); diff --git a/src/parakeet_capi.cpp b/src/parakeet_capi.cpp index 01de213..77e163a 100644 --- a/src/parakeet_capi.cpp +++ b/src/parakeet_capi.cpp @@ -24,7 +24,14 @@ // surface per-word timestamps (start/end/conf) plus frame_sec alongside the // newly-finalized text + eou flag, and a "frame_sec" field added to the // transcribe_*_json documents. Original entry points unchanged. -#define PARAKEET_CAPI_ABI_VERSION 4 +// v5: vs distinction across the C boundary. BREAKING semantics: +// stream_feed's *eou_out is now a bitmask (PARAKEET_EVENT_EOU | +// PARAKEET_EVENT_EOB) instead of an any-event 0/1, and the JSON "eou" +// field now means "an fired" only, with a new "eob" field beside it. +// Added stream_drain_events / free_events (typed per-event records) and +// the "events" array in the stream_feed_json / stream_finalize_json +// documents. +#define PARAKEET_CAPI_ABI_VERSION 5 // The opaque context: a loaded model plus a buffer for the last error message. struct parakeet_ctx { @@ -137,6 +144,13 @@ void append_json_string(std::string& out, const std::string& s) { out += '"'; } +// Append an int to `out` as a bare JSON number. +void append_json_int(std::string& out, int v) { + char buf[16]; + std::snprintf(buf, sizeof(buf), "%d", v); + out += buf; +} + // Append a float to `out` formatted with `fmt` (e.g. "%.3f"). NaN/Inf are // emitted as 0 (JSON has no NaN/Inf literal); confidences/times are finite here. void append_json_float(std::string& out, const char* fmt, float v) { @@ -178,10 +192,8 @@ std::string transcription_to_json(const pk::Transcription& tr, float frame_sec) out += "],\"tokens\":["; for (size_t i = 0; i < tr.tokens.size(); ++i) { if (i) out += ','; - char idbuf[16]; - std::snprintf(idbuf, sizeof(idbuf), "%d", tr.tokens[i].id); out += "{\"id\":"; - out += idbuf; + append_json_int(out, tr.tokens[i].id); out += ",\"t\":"; append_json_float(out, "%.3f", (float)tr.tokens[i].frame * frame_sec); out += ",\"conf\":"; @@ -428,11 +440,16 @@ namespace { // new PCM is produced frame-local by StreamingMel in the feed/finalize entry // points, NOT recomputed over the whole buffer here. `flush` marks the final // partial chunk is_last (keep_all_outputs), draining the remaining frames. Sets -// *eou to 1 if an / event fired in this pass. Returns the newly- +// eou_flag / eob_flag to 1 if an / (respectively) fired in this +// pass — attributed by watermarking the session's un-drained event queue, so +// the queue itself is left intact for the caller to drain. Returns the newly- // finalized text. -std::string feed_available(parakeet_stream* s, bool flush, int& eou_flag) { +std::string feed_available(parakeet_stream* s, bool flush, int& eou_flag, + int& eob_flag) { eou_flag = 0; + eob_flag = 0; pk::StreamingSession& sess = *s->sess; + const size_t ev0 = sess.events().size(); const int n_mels = s->n_mels; const int T = s->mel_T; @@ -474,12 +491,13 @@ std::string feed_available(parakeet_stream* s, bool flush, int& eou_flag) { sess.feed_mel_chunk(win, win_frames, is_last); new_text += sess.take_new_text(); - if (sess.last_chunk_had_eou()) eou_flag = 1; s->mel_buffer_idx += chunk_size; // shift_size == chunk_size here s->first_chunk = false; if (is_last) break; // flushed the end-of-stream tail } + for (size_t i = ev0; i < sess.events().size(); ++i) + (sess.events()[i].is_eob ? eob_flag : eou_flag) = 1; return new_text; } @@ -536,9 +554,10 @@ extern "C" char* parakeet_capi_stream_feed(parakeet_stream* s, const float* pcm, std::vector frames = s->mel->feed(pcm, n_samples, n_new); append_mel_frames(s, frames, n_new); } - int eou = 0; - std::string delta = feed_available(s, /*flush=*/false, eou); - if (eou_out) *eou_out = eou; + int eou = 0, eob = 0; + std::string delta = feed_available(s, /*flush=*/false, eou, eob); + if (eou_out) *eou_out = (eou ? PARAKEET_EVENT_EOU : 0) | + (eob ? PARAKEET_EVENT_EOB : 0); s->ctx->last_error.clear(); char* out = dup_to_c(delta); if (!out) { s->ctx->last_error = "out of memory"; return nullptr; } @@ -563,8 +582,8 @@ extern "C" char* parakeet_capi_stream_finalize(parakeet_stream* s) { std::vector tail = s->mel->finalize(n_tail); append_mel_frames(s, tail, n_tail); } - int eou = 0; - std::string delta = feed_available(s, /*flush=*/true, eou); + int eou = 0, eob = 0; + std::string delta = feed_available(s, /*flush=*/true, eou, eob); // After the flush the session's finalize() is a no-op text-wise (no extra // audio) but documents the end-of-stream tail semantics. delta += s->sess->finalize(); @@ -582,22 +601,74 @@ extern "C" char* parakeet_capi_stream_finalize(parakeet_stream* s) { } } +extern "C" int parakeet_capi_stream_drain_events(parakeet_stream* s, + parakeet_stream_event** out_events) { + if (out_events) *out_events = nullptr; + if (!s || !out_events) return -1; + if (!s->ctx || !s->ctx->model || !s->sess) return -1; + try { + std::vector evs = s->sess->drain_events(); + s->ctx->last_error.clear(); + if (evs.empty()) return 0; + auto* arr = static_cast( + std::malloc(evs.size() * sizeof(parakeet_stream_event))); + if (!arr) { s->ctx->last_error = "out of memory"; return -1; } + for (size_t i = 0; i < evs.size(); ++i) { + arr[i].token = (int)evs[i].token; + arr[i].is_eob = evs[i].is_eob ? 1 : 0; + arr[i].encoder_frame = evs[i].encoder_frame; + arr[i].time_sec = (float)evs[i].time_sec; + } + *out_events = arr; + return (int)evs.size(); + } catch (const std::exception& e) { + s->ctx->last_error = e.what(); + return -1; + } catch (...) { + s->ctx->last_error = "unknown error"; + return -1; + } +} + +extern "C" void parakeet_capi_free_events(parakeet_stream_event* events) { + std::free(events); +} + namespace { // Serialize a streaming feed/finalize result to JSON: the newly-finalized text, -// the eou flag, frame_sec, and the words drained this call (absolute seconds). -// Shape matches the header doc on parakeet_capi_stream_feed_json. -std::string stream_json(const std::string& text, int eou, float frame_sec, +// the per-type eou/eob flags, frame_sec, the / events drained this +// call, and the words drained this call (absolute seconds). Shape matches the +// header doc on parakeet_capi_stream_feed_json. "eou" means an fired and +// "eob" an — they are NOT conflated (a voice agent responds on eou and +// must not treat eob as the user taking the turn); "events" carries the +// per-event timestamps. +std::string stream_json(const std::string& text, int eou, int eob, + float frame_sec, + const std::vector& events, const std::vector& words) { std::string out; - out.reserve(80 + words.size() * 48); + out.reserve(80 + events.size() * 36 + words.size() * 48); out += "{\"text\":"; append_json_string(out, text); out += ",\"eou\":"; out += (eou ? "1" : "0"); + out += ",\"eob\":"; + out += (eob ? "1" : "0"); out += ",\"frame_sec\":"; append_json_float(out, "%.6f", frame_sec); - out += ",\"words\":["; + out += ",\"events\":["; + for (size_t i = 0; i < events.size(); ++i) { + if (i) out += ','; + out += "{\"type\":"; + out += events[i].is_eob ? "\"eob\"" : "\"eou\""; + out += ",\"frame\":"; + append_json_int(out, events[i].encoder_frame); + out += ",\"t\":"; + append_json_float(out, "%.3f", (float)events[i].time_sec); + out += '}'; + } + out += "],\"words\":["; for (size_t i = 0; i < words.size(); ++i) { if (i) out += ','; out += "{\"w\":"; @@ -636,10 +707,11 @@ extern "C" char* parakeet_capi_stream_feed_json(parakeet_stream* s, std::vector frames = s->mel->feed(pcm, n_samples, n_new); append_mel_frames(s, frames, n_new); } - int eou = 0; - std::string delta = feed_available(s, /*flush=*/false, eou); + int eou = 0, eob = 0; + std::string delta = feed_available(s, /*flush=*/false, eou, eob); + std::vector events = s->sess->drain_events(); std::vector words = s->sess->drain_words(); - std::string json = stream_json(delta, eou, stream_frame_sec(s), words); + std::string json = stream_json(delta, eou, eob, stream_frame_sec(s), events, words); s->ctx->last_error.clear(); char* out = dup_to_c(json); if (!out) { s->ctx->last_error = "out of memory"; return nullptr; } @@ -662,11 +734,12 @@ extern "C" char* parakeet_capi_stream_finalize_json(parakeet_stream* s) { std::vector tail = s->mel->finalize(n_tail); append_mel_frames(s, tail, n_tail); } - int eou = 0; - std::string delta = feed_available(s, /*flush=*/true, eou); + int eou = 0, eob = 0; + std::string delta = feed_available(s, /*flush=*/true, eou, eob); delta += s->sess->finalize(); + std::vector events = s->sess->drain_events(); std::vector words = s->sess->drain_words(); - std::string json = stream_json(delta, eou, stream_frame_sec(s), words); + std::string json = stream_json(delta, eou, eob, stream_frame_sec(s), events, words); s->finalized = true; s->ctx->last_error.clear(); char* out = dup_to_c(json); diff --git a/src/streaming.hpp b/src/streaming.hpp index 083c2db..1480519 100644 --- a/src/streaming.hpp +++ b/src/streaming.hpp @@ -109,6 +109,11 @@ class StreamingSession { // Move out all EOU/EOB events collected so far (drains the queue). std::vector drain_events(); + // Peek at the not-yet-drained EOU/EOB events without consuming them (the + // C-API uses the size as a watermark to attribute events to one feed pass + // while leaving the queue intact for a later drain). + const std::vector& events() const { return events_; } + // Move out the WORDS finalized since the previous drain_words() call, with // per-word start/end (seconds) and 'min'-aggregate confidence (matching the // offline pk::group_words / NeMo timestamps=True convention). A word is diff --git a/tests/stream_clips.hpp b/tests/stream_clips.hpp new file mode 100644 index 0000000..17b2989 --- /dev/null +++ b/tests/stream_clips.hpp @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace pktest { + +// Build the two-utterance clip used by the streaming event tests: the input +// clip + 0.6 s of silence + the input clip again, at 16 kHz. This is the same +// construction (and the same default gap) as scripts/gen_stream_reset_baseline.py, +// so a mid-stream fires between the two utterances. +inline std::vector two_utterance_clip(const std::vector& samples) { + std::vector clip(samples); + clip.insert(clip.end(), (size_t)(0.6f * 16000.0f), 0.0f); + clip.insert(clip.end(), samples.begin(), samples.end()); + return clip; +} + +} // namespace pktest diff --git a/tests/test_capi_stream.cpp b/tests/test_capi_stream.cpp index 05fb453..158df93 100644 --- a/tests/test_capi_stream.cpp +++ b/tests/test_capi_stream.cpp @@ -1,6 +1,7 @@ #include "parakeet_capi.h" -#include "audio_io.hpp" // pk::load_audio_16k_mono (test links the parakeet lib) -#include "parity.hpp" // pktest::load_kv_str / load_baseline_i32 +#include "audio_io.hpp" // pk::load_audio_16k_mono (test links the parakeet lib) +#include "parity.hpp" // pktest::load_kv_str / load_baseline_i32 +#include "stream_clips.hpp" // pktest::two_utterance_clip #include #include @@ -115,7 +116,6 @@ int main() { parakeet_capi_free_string(fin); parakeet_capi_stream_free(s); - parakeet_capi_free(ctx); std::fprintf(stderr, "test_capi_stream: got stream_text = %s\n", acc.c_str()); std::fprintf(stderr, "test_capi_stream: any in-stream EOU event = %s\n", @@ -128,13 +128,113 @@ int main() { " got: %s\n" " expected: %s\n", acc.c_str(), ref_text.c_str()); + parakeet_capi_free(ctx); return 1; } + // --- Phase 2: drain_events on a TWO-utterance clip (ABI v5) --- + // An fires mid-stream (see stream_clips.hpp). The typed drain must + // surface it with is_eob==0 and a sane absolute timestamp; the per-feed + // event mask and the drained queue must agree. + { + std::vector clip = pktest::two_utterance_clip(audio.samples); + + parakeet_stream* s2 = parakeet_capi_stream_begin(ctx); + if (!s2) { + std::fprintf(stderr, "test_capi_stream: phase-2 stream_begin failed: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_free(ctx); + return 1; + } + int flagged_feeds = 0; + std::vector drained; + const int n2 = (int)clip.size(); + for (int off = 0; off < n2; off += chunk) { + const int len = std::min(chunk, n2 - off); + int eou = 0; + char* t = parakeet_capi_stream_feed(s2, clip.data() + off, len, &eou); + if (!t) { + std::fprintf(stderr, "test_capi_stream: phase-2 feed NULL: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_stream_free(s2); + parakeet_capi_free(ctx); + return 1; + } + parakeet_capi_free_string(t); + parakeet_stream_event* evs = nullptr; + const int n_ev = parakeet_capi_stream_drain_events(s2, &evs); + if (n_ev < 0) { + std::fprintf(stderr, "test_capi_stream: drain_events errored: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_stream_free(s2); + parakeet_capi_free(ctx); + return 1; + } + // The bare feed's event mask and the typed queue must agree per feed. + int want_mask = 0; + for (int i = 0; i < n_ev; ++i) + want_mask |= evs[i].is_eob ? PARAKEET_EVENT_EOB : PARAKEET_EVENT_EOU; + if (eou != want_mask) { + std::fprintf(stderr, + "test_capi_stream: FAIL — feed event mask (%d) disagrees with " + "drained events (want %d from %d events)\n", eou, want_mask, n_ev); + parakeet_capi_free_events(evs); + parakeet_capi_stream_free(s2); + parakeet_capi_free(ctx); + return 1; + } + if (eou) ++flagged_feeds; + drained.insert(drained.end(), evs, evs + n_ev); + parakeet_capi_free_events(evs); + } + char* fin2 = parakeet_capi_stream_finalize(s2); + if (fin2) parakeet_capi_free_string(fin2); + { + // Events from the finalize flush (e.g. a trailing end-of-clip ) + // land in the same queue. + parakeet_stream_event* evs = nullptr; + const int n_ev = parakeet_capi_stream_drain_events(s2, &evs); + if (n_ev > 0) drained.insert(drained.end(), evs, evs + n_ev); + parakeet_capi_free_events(evs); + } + parakeet_capi_stream_free(s2); + + const float clip_secs = (float)n2 / 16000.0f; + if (drained.empty() || flagged_feeds < 1) { + std::fprintf(stderr, + "test_capi_stream: FAIL — no event drained on the " + "two-utterance clip (flagged feeds=%d)\n", flagged_feeds); + parakeet_capi_free(ctx); + return 1; + } + float prev_t = -1.0f; + for (const parakeet_stream_event& e : drained) { + const bool sane = (e.is_eob == 0 || e.is_eob == 1) && + e.encoder_frame >= 0 && + e.time_sec >= prev_t && + e.time_sec <= clip_secs + 1.0f; + if (!sane) { + std::fprintf(stderr, + "test_capi_stream: FAIL — bad event record (token=%d is_eob=%d " + "frame=%d t=%.3f, clip=%.2fs)\n", + e.token, e.is_eob, e.encoder_frame, e.time_sec, clip_secs); + parakeet_capi_free(ctx); + return 1; + } + prev_t = e.time_sec; + std::fprintf(stderr, "test_capi_stream: event %s frame=%d t=%.3fs\n", + e.is_eob ? "" : "", e.encoder_frame, e.time_sec); + } + } + + parakeet_capi_free(ctx); + std::fprintf(stderr, "test_capi_stream: PASS — streaming C-API transcript == NeMo cache-aware " "streaming decode (%zu ref tokens, EOU stripped + surfaced as events; " - "no in-stream for this clip, matching NeMo eou_in_stream=0)\n", + "no in-stream for this clip, matching NeMo eou_in_stream=0), and " + "drain_events surfaces typed / records on the two-utterance " + "clip\n", ref_ids.size()); return 0; } diff --git a/tests/test_capi_stream_json.cpp b/tests/test_capi_stream_json.cpp index 785a37e..17e59fe 100644 --- a/tests/test_capi_stream_json.cpp +++ b/tests/test_capi_stream_json.cpp @@ -1,5 +1,6 @@ #include "parakeet_capi.h" -#include "audio_io.hpp" // pk::load_audio_16k_mono (test links the parakeet lib) +#include "audio_io.hpp" // pk::load_audio_16k_mono (test links the parakeet lib) +#include "stream_clips.hpp" // pktest::two_utterance_clip #include #include @@ -83,25 +84,90 @@ int main() { parakeet_capi_free_string(fin); parakeet_capi_stream_free(s); - parakeet_capi_free(ctx); std::fprintf(stderr, "test_capi_stream_json: concatenated docs:\n%s\n", acc.c_str()); if (!contains(acc, "\"frame_sec\"")) { std::fprintf(stderr, "test_capi_stream_json: FAIL — no frame_sec in output\n"); + parakeet_capi_free(ctx); return 1; } if (!contains(acc, "\"words\"")) { std::fprintf(stderr, "test_capi_stream_json: FAIL — no words array in output\n"); + parakeet_capi_free(ctx); return 1; } // A real transcription must finalize at least one word with a non-zero end. if (!contains(acc, "\"end\":")) { std::fprintf(stderr, "test_capi_stream_json: FAIL — no word end timestamps\n"); + parakeet_capi_free(ctx); + return 1; + } + // ABI v5: every streaming doc carries the (possibly empty) "events" array + // and the per-type "eou"/"eob" flags. + if (!contains(acc, "\"events\"")) { + std::fprintf(stderr, "test_capi_stream_json: FAIL — no events array in output\n"); + parakeet_capi_free(ctx); return 1; } + if (!contains(acc, "\"eou\":") || !contains(acc, "\"eob\":")) { + std::fprintf(stderr, "test_capi_stream_json: FAIL — missing eou/eob flags\n"); + parakeet_capi_free(ctx); + return 1; + } + + // Two-utterance clip (see stream_clips.hpp): an fires mid-stream and + // must appear as a typed entry {"type":"eou","frame":...,"t":...} in some doc. + { + std::vector clip = pktest::two_utterance_clip(audio.samples); + + parakeet_stream* s2 = parakeet_capi_stream_begin(ctx); + if (!s2) { + std::fprintf(stderr, "test_capi_stream_json: phase-2 stream_begin failed: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_free(ctx); + return 1; + } + std::string acc2; + const int n2 = (int)clip.size(); + for (int off = 0; off < n2; off += chunk) { + const int len = std::min(chunk, n2 - off); + char* t = parakeet_capi_stream_feed_json(s2, clip.data() + off, len); + if (!t) { + std::fprintf(stderr, "test_capi_stream_json: phase-2 feed_json NULL: %s\n", + parakeet_capi_last_error(ctx)); + parakeet_capi_stream_free(s2); + parakeet_capi_free(ctx); + return 1; + } + acc2 += t; + parakeet_capi_free_string(t); + } + char* fin2 = parakeet_capi_stream_finalize_json(s2); + if (fin2) { acc2 += fin2; parakeet_capi_free_string(fin2); } + parakeet_capi_stream_free(s2); + + if (!contains(acc2, "\"type\":\"eou\"")) { + std::fprintf(stderr, + "test_capi_stream_json: FAIL — no typed eou event on the " + "two-utterance clip; docs:\n%s\n", acc2.c_str()); + parakeet_capi_free(ctx); + return 1; + } + // The per-type flag fires with the event (and EOB stays 0 — this clip + // has no backchannel). + if (!contains(acc2, "\"eou\":1") || contains(acc2, "\"eob\":1")) { + std::fprintf(stderr, + "test_capi_stream_json: FAIL — eou/eob flags wrong on the " + "two-utterance clip; docs:\n%s\n", acc2.c_str()); + parakeet_capi_free(ctx); + return 1; + } + } + + parakeet_capi_free(ctx); std::fprintf(stderr, "test_capi_stream_json: PASS — streaming JSON carries " - "frame_sec + per-word timestamps\n"); + "frame_sec + per-word timestamps + typed eou/eob events\n"); return 0; }