diff --git a/docs/react-hooks.md b/docs/react-hooks.md index 8ac18f4..3f06d3d 100644 --- a/docs/react-hooks.md +++ b/docs/react-hooks.md @@ -99,6 +99,11 @@ useTranscription((entry) => { }, { interim: false }); ``` +Runway streaming transcripts are combined by role and turn. The worker's +`final: true` marker completes a turn immediately, including the last turn in a +session. Older workers without that marker are still handled when the next +turn for the same role begins. + ## useClientEvent Subscribe to typed client events from the avatar: diff --git a/packages/core/src/client-transcript.test.ts b/packages/core/src/client-transcript.test.ts new file mode 100644 index 0000000..7d1b452 --- /dev/null +++ b/packages/core/src/client-transcript.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'bun:test'; +import { Room, RoomEvent } from 'livekit-client'; +import type { RemoteParticipant } from 'livekit-client'; +import { AvatarSession } from './client'; + +describe('AvatarSession transcript event routing', () => { + it('keeps final-only transcript entries stable across role changes and late deltas', () => { + const session = new AvatarSession('test-session'); + const room = new Room({ adaptiveStream: false, dynacast: false }); + const bindRoomEvents = (session as unknown as { + bindRoomEvents: (room: Room) => void; + }).bindRoomEvents; + bindRoomEvents.call(session, room); + + const transcript = session.transcript(); + const updates: Array> = []; + transcript.on('update', (entries) => { + updates.push( + entries.map(({ id, text, final }) => ({ id, text, final })), + ); + }); + + const send = (payload: Record, identity: string) => { + room.emit( + RoomEvent.DataReceived, + new TextEncoder().encode(JSON.stringify(payload)), + { identity } as RemoteParticipant, + ); + }; + + try { + send( + { type: 'transcription', role: 'assistant', turn: 0, text: 'Hello' }, + 'worker:avatar', + ); + expect(transcript.entries).toEqual([]); + expect(updates).toEqual([]); + + send( + { type: 'transcription', role: 'user', turn: 0, text: 'Hi' }, + 'participant:user', + ); + send( + { + type: 'transcription', + role: 'user', + turn: 0, + text: '', + final: true, + }, + 'participant:user', + ); + + expect(transcript.entries).toEqual([ + expect.objectContaining({ + id: 'runway-transcription-user-0', + text: 'Hi', + final: true, + participantIdentity: 'participant:user', + }), + ]); + + send( + { + type: 'transcription', + role: 'assistant', + turn: 0, + text: '', + final: true, + }, + 'worker:avatar', + ); + expect(transcript.entries).toEqual([ + expect.objectContaining({ + id: 'runway-transcription-user-0', + text: 'Hi', + final: true, + }), + expect.objectContaining({ + id: 'runway-transcription-assistant-0', + text: 'Hello', + final: true, + participantIdentity: 'worker:avatar', + }), + ]); + + const callbackCountAfterFinals = updates.length; + send( + { + type: 'transcription', + role: 'assistant', + turn: 0, + text: '', + final: true, + }, + 'worker:avatar', + ); + send( + { type: 'transcription', role: 'assistant', turn: 0, text: ' late' }, + 'worker:avatar', + ); + + expect(updates).toHaveLength(callbackCountAfterFinals); + expect(transcript.entries).toEqual([ + expect.objectContaining({ + id: 'runway-transcription-user-0', + text: 'Hi', + final: true, + }), + expect.objectContaining({ + id: 'runway-transcription-assistant-0', + text: 'Hello', + final: true, + }), + ]); + } finally { + transcript.dispose(); + room.removeAllListeners(); + } + }); +}); diff --git a/packages/core/src/client.ts b/packages/core/src/client.ts index 42eb14c..70d5c54 100644 --- a/packages/core/src/client.ts +++ b/packages/core/src/client.ts @@ -368,11 +368,13 @@ export class AvatarSession extends Emitter { channel: 'custom', }); } - this.emit(AvatarEvent.Transcript, { - ...active, - final: false, - channel: 'custom', - }); + if (!active.final) { + this.emit(AvatarEvent.Transcript, { + ...active, + final: false, + channel: 'custom', + }); + } } }, ); diff --git a/packages/core/src/transcript-accumulator.test.ts b/packages/core/src/transcript-accumulator.test.ts index 30c3f86..d714b30 100644 --- a/packages/core/src/transcript-accumulator.test.ts +++ b/packages/core/src/transcript-accumulator.test.ts @@ -101,6 +101,61 @@ describe('TranscriptAccumulator', () => { expect(acc.entries[0].text).toBe('Hello'); }); + it('exposes the last flat turn as final without interim mode', () => { + const acc = new TranscriptAccumulator(); + const encode = (obj: object) => + new TextEncoder().encode(JSON.stringify(obj)); + + acc.ingestDataChannel( + encode({ type: 'transcription', role: 'assistant', turn: 3, text: 'last answer' }), + 'worker:avatar', + ); + expect(acc.entries).toEqual([]); + + acc.ingestDataChannel( + encode({ + type: 'transcription', + role: 'assistant', + turn: 3, + text: '', + final: true, + }), + 'worker:avatar', + ); + + expect(acc.entries).toHaveLength(1); + expect(acc.entries[0]).toMatchObject({ + id: 'runway-transcription-assistant-3', + text: 'last answer', + final: true, + channel: 'custom', + }); + }); + + it('does not let an interim update overwrite a final flat entry', () => { + const acc = new TranscriptAccumulator({ interim: true }); + const encode = (obj: object) => + new TextEncoder().encode(JSON.stringify(obj)); + + acc.ingestDataChannel( + encode({ + type: 'transcription', + role: 'assistant', + turn: 0, + text: 'done', + final: true, + }), + 'worker:avatar', + ); + acc.ingestDataChannel( + encode({ type: 'transcription', role: 'assistant', turn: 0, text: ' late' }), + 'worker:avatar', + ); + + expect(acc.entries).toHaveLength(1); + expect(acc.entries[0]).toMatchObject({ text: 'done', final: true }); + }); + it('dispose clears state and removes listeners', () => { const acc = new TranscriptAccumulator(); const handler = mock(); diff --git a/packages/core/src/transcript-accumulator.ts b/packages/core/src/transcript-accumulator.ts index 269fbff..86afa1d 100644 --- a/packages/core/src/transcript-accumulator.ts +++ b/packages/core/src/transcript-accumulator.ts @@ -100,7 +100,7 @@ export class TranscriptAccumulator extends Emitter { changed = true; } - if (this.interim) { + if (this.interim && !active.final) { this.map.set(active.id, { ...active, final: false, channel: 'custom' }); changed = true; } diff --git a/packages/core/src/utils/flatDeltaAccumulator.test.ts b/packages/core/src/utils/flatDeltaAccumulator.test.ts index 916bf73..4070112 100644 --- a/packages/core/src/utils/flatDeltaAccumulator.test.ts +++ b/packages/core/src/utils/flatDeltaAccumulator.test.ts @@ -47,6 +47,93 @@ describe('FlatDeltaAccumulator', () => { }); }); + it('finalizes the active turn immediately on an explicit completion marker', () => { + const acc = new FlatDeltaAccumulator(); + acc.ingest({ role: 'assistant', turn: 0, textDelta: 'hello ' }, 'worker:1'); + + const emission = acc.ingest( + { role: 'assistant', turn: 0, textDelta: 'there', final: true }, + 'worker:1', + ); + + expect(emission.finalized).toEqual([ + { + id: 'runway-transcription-assistant-0', + text: 'hello there', + participantIdentity: 'worker:1', + }, + ]); + expect(emission.active).toMatchObject({ + id: 'runway-transcription-assistant-0', + text: 'hello there', + final: true, + }); + }); + + it('does not create a visible empty entry for an unobserved completion marker', () => { + const acc = new FlatDeltaAccumulator(); + const emission = acc.ingest( + { role: 'assistant', turn: 0, textDelta: '', final: true }, + 'worker:1', + ); + + expect(emission.finalized).toEqual([]); + expect(emission.active).toMatchObject({ text: '', final: true }); + + const late = acc.ingest( + { role: 'assistant', turn: 0, textDelta: 'late' }, + 'worker:1', + ); + expect(late.finalized).toEqual([]); + expect(late.active).toMatchObject({ text: '', final: true }); + }); + + it('does not reopen a finalized turn when a late delta arrives', () => { + const acc = new FlatDeltaAccumulator(); + acc.ingest({ role: 'assistant', turn: 0, textDelta: 'done', final: true }, 'worker:1'); + + const late = acc.ingest( + { role: 'assistant', turn: 0, textDelta: ' late' }, + 'worker:1', + ); + expect(late.finalized).toEqual([]); + expect(late.active).toMatchObject({ text: 'done', final: true }); + }); + + it('does not infer finality for a newer turn from an out-of-order older delta', () => { + const acc = new FlatDeltaAccumulator(); + acc.ingest({ role: 'assistant', turn: 2, textDelta: 'newer' }, 'worker:1'); + + const older = acc.ingest( + { role: 'assistant', turn: 1, textDelta: 'older' }, + 'worker:1', + ); + expect(older.finalized).toEqual([]); + }); + + it('finalizes roles independently when explicit completion markers arrive', () => { + const acc = new FlatDeltaAccumulator(); + acc.ingest({ role: 'assistant', turn: 0, textDelta: 'answer' }, 'worker:1'); + acc.ingest({ role: 'user', turn: 0, textDelta: 'question' }, 'participant:me'); + + const assistant = acc.ingest( + { role: 'assistant', turn: 0, textDelta: '', final: true }, + 'worker:1', + ); + const duplicateAssistant = acc.ingest( + { role: 'assistant', turn: 0, textDelta: '', final: true }, + 'worker:1', + ); + const user = acc.ingest( + { role: 'user', turn: 0, textDelta: '', final: true }, + 'participant:me', + ); + + expect(assistant.finalized.map((entry) => entry.text)).toEqual(['answer']); + expect(duplicateAssistant.finalized).toEqual([]); + expect(user.finalized.map((entry) => entry.text)).toEqual(['question']); + }); + it('emits a finalized turn exactly once across subsequent deltas', () => { const acc = new FlatDeltaAccumulator(); diff --git a/packages/core/src/utils/flatDeltaAccumulator.ts b/packages/core/src/utils/flatDeltaAccumulator.ts index 34f56c4..3b629ae 100644 --- a/packages/core/src/utils/flatDeltaAccumulator.ts +++ b/packages/core/src/utils/flatDeltaAccumulator.ts @@ -2,6 +2,7 @@ export interface FlatDelta { role: string; turn: number; textDelta: string; + final?: boolean; } export interface FlatDeltaTurn { @@ -13,17 +14,15 @@ export interface FlatDeltaTurn { export interface FlatDeltaEmission { /** Prior turns on the same role that were just promoted to final by this delta. */ finalized: Array; - /** The turn this delta belongs to — still streaming, always interim. */ - active: FlatDeltaTurn; + /** The turn this delta belongs to, with `final` set for completed markers. */ + active: FlatDeltaTurn & { final?: boolean }; } /** * Accumulates Runway flat-delta transcription fragments into coherent turns. * - * The flat-delta protocol sends `{ role, turn, textDelta }` messages without - * an explicit end-of-turn signal, so we infer the end: when a delta arrives - * on a higher turn for the same role, prior turns on that role are done and - * get emitted once as final. The active turn is always interim. + * The flat-delta protocol may send an explicit `final` marker. Older workers + * omit it, so we also infer completion when a higher turn arrives for a role. * * Identity is remembered per-turn, so finalized priors are attributed to * whoever originally produced them — not to whoever triggered the role @@ -32,22 +31,47 @@ export interface FlatDeltaEmission { export class FlatDeltaAccumulator { private readonly turns = new Map< string, - { text: string; participantIdentity: string } + { turn: number; text: string; participantIdentity: string } >(); private readonly finalized = new Set(); ingest(delta: FlatDelta, participantIdentity: string): FlatDeltaEmission { const id = `runway-transcription-${delta.role}-${delta.turn}`; const prev = this.turns.get(id); + + // Final turns are immutable. This also makes duplicate final markers and + // late deltas harmless after a turn has been committed. + if (this.finalized.has(id)) { + return { + finalized: [], + active: { + id, + text: prev?.text ?? '', + participantIdentity: prev?.participantIdentity ?? participantIdentity, + final: true, + }, + }; + } + const text = (prev?.text ?? '') + delta.textDelta; const identity = prev?.participantIdentity ?? participantIdentity; - this.turns.set(id, { text, participantIdentity: identity }); + const hasTranscript = prev !== undefined || text.length > 0; + if (hasTranscript) { + this.turns.set(id, { turn: delta.turn, text, participantIdentity: identity }); + } const prefix = `runway-transcription-${delta.role}-`; const finalized: Array = []; - for (const [key, turn] of this.turns) { - if (key === id || !key.startsWith(prefix)) continue; - if (this.finalized.has(key)) continue; + const priorTurns = Array.from(this.turns.entries()) + .filter( + ([key, turn]) => + key !== id && + key.startsWith(prefix) && + turn.turn < delta.turn && + !this.finalized.has(key), + ) + .sort(([, first], [, second]) => first.turn - second.turn); + for (const [key, turn] of priorTurns) { this.finalized.add(key); finalized.push({ id: key, @@ -56,9 +80,19 @@ export class FlatDeltaAccumulator { }); } + if (delta.final) { + this.finalized.add(id); + if (hasTranscript) finalized.push({ id, text, participantIdentity: identity }); + } + return { finalized, - active: { id, text, participantIdentity: identity }, + active: { + id, + text, + participantIdentity: identity, + ...(delta.final ? { final: true } : {}), + }, }; } diff --git a/packages/core/src/utils/parseTranscription.test.ts b/packages/core/src/utils/parseTranscription.test.ts index b8daa66..baa219e 100644 --- a/packages/core/src/utils/parseTranscription.test.ts +++ b/packages/core/src/utils/parseTranscription.test.ts @@ -52,6 +52,66 @@ describe('tryParseFlatDelta', () => { }); }); + it('preserves an optional final marker', () => { + expect( + tryParseFlatDelta({ + type: 'transcription', + role: 'assistant', + turn: 2, + text: '', + final: true, + }), + ).toEqual({ + role: 'assistant', + turn: 2, + textDelta: '', + final: true, + }); + }); + + it('ignores completion markers without a known role', () => { + expect( + tryParseFlatDelta({ + type: 'transcription', + turn: 2, + text: '', + final: true, + }), + ).toBeNull(); + expect( + tryParseFlatDelta({ + type: 'transcription', + role: '', + turn: 2, + text: '', + final: true, + }), + ).toBeNull(); + expect( + tryParseFlatDelta({ + type: 'transcription', + role: 'tool', + turn: 2, + text: '', + final: true, + }), + ).toBeNull(); + }); + + it('ignores completion markers without a valid explicit turn', () => { + for (const turn of [undefined, NaN, Infinity, 1.5, -1]) { + expect( + tryParseFlatDelta({ + type: 'transcription', + role: 'assistant', + turn, + text: '', + final: true, + }), + ).toBeNull(); + } + }); + it('defaults role to assistant and turn to 0 when missing', () => { const result = tryParseFlatDelta({ type: 'transcription', text: 'hi' }); expect(result).toEqual({ role: 'assistant', turn: 0, textDelta: 'hi' }); diff --git a/packages/core/src/utils/parseTranscription.ts b/packages/core/src/utils/parseTranscription.ts index c7b1ff8..5129042 100644 --- a/packages/core/src/utils/parseTranscription.ts +++ b/packages/core/src/utils/parseTranscription.ts @@ -69,13 +69,31 @@ export function tryParseSegmentArray( */ export function tryParseFlatDelta( root: DataChannelJSON, -): { role: string; turn: number; textDelta: string } | null { +): { role: string; turn: number; textDelta: string; final?: boolean } | null { if (root.type !== 'transcription') return null; if (typeof root.text !== 'string') return null; + const final = typeof root.final === 'boolean' ? root.final : undefined; + // A completion marker must identify a role. Keep the historical assistant + // default for ordinary deltas, but do not turn malformed markers into data. + if (final) { + const role = root.role; + const turn = root.turn; + if ( + (role !== 'assistant' && role !== 'user') || + typeof turn !== 'number' || + !Number.isFinite(turn) || + !Number.isInteger(turn) || + turn < 0 + ) { + return null; + } + } + return { role: typeof root.role === 'string' ? root.role : 'assistant', turn: typeof root.turn === 'number' ? root.turn : 0, textDelta: root.text, + ...(final !== undefined ? { final } : {}), }; } diff --git a/packages/react/src/hooks/useTranscript.ts b/packages/react/src/hooks/useTranscript.ts index deb969b..bbf7527 100644 --- a/packages/react/src/hooks/useTranscript.ts +++ b/packages/react/src/hooks/useTranscript.ts @@ -21,9 +21,10 @@ export interface UseTranscriptOptions { * Also merge transcript JSON from `RoomEvent.DataReceived` (data channel). * Covers LiveKit-style `{ segments: [{ id, text, … }] }` and Runway worker * `{ type: "transcription", role, turn, text }` streaming deltas - * (concatenated per role+turn). When a new turn starts on the same role, - * the prior turn is promoted to `final: true`. The active (streaming) turn - * is always interim and only surfaced when `interim: true`. + * (concatenated per role+turn). A `final: true` marker completes that turn; + * older workers are also handled by promoting prior turns when a newer turn + * starts on the same role. The active turn is only surfaced when + * `interim: true`. * Default: `true`. */ mergeDataChannelSegments?: boolean; @@ -140,9 +141,8 @@ export function useTranscript( changed = true; } - // The active turn is still streaming, so it's always interim — only - // surface it when the caller opted in with `interim: true`. - if (interimRef.current) { + // The active turn is streaming unless this delta carries a final marker. + if (interimRef.current && !active.final) { mapRef.current.set(active.id, { ...active, final: false, diff --git a/packages/react/src/hooks/useTranscription.ts b/packages/react/src/hooks/useTranscription.ts index d7f2b11..15b6707 100644 --- a/packages/react/src/hooks/useTranscription.ts +++ b/packages/react/src/hooks/useTranscription.ts @@ -24,9 +24,10 @@ import { * transport the backend uses. Each callback entry includes `channel`: `native` * or `custom` accordingly. * - * For Runway flat-delta transcripts (custom channel), the active turn is - * always interim and only delivered when `interim: true`. A prior turn is - * delivered once as `final: true` when a new turn on the same role begins. + * For Runway flat-delta transcripts (custom channel), a `final: true` marker + * delivers the completed turn immediately. Older workers are also handled by + * delivering a prior turn when a new turn on the same role begins. Interim + * updates are only delivered when `interim: true`. * * Must be used within an AvatarSession, AvatarProvider, or AvatarCall component. * @@ -100,8 +101,8 @@ export function useTranscription( handlerRef.current({ ...turn, final: true, channel: 'custom' }); } - // Active turn is still streaming — only emit when caller opted in. - if (interimRef.current) { + // Active turn is streaming unless this delta carries a final marker. + if (interimRef.current && !active.final) { handlerRef.current({ ...active, final: false, channel: 'custom' }); } }