Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/react-hooks.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ useTranscription((entry) => {
}, { interim: false });
```

Runway streaming transcripts are combined by role and turn. The worker's
`final: true` marker completes a turn immediately, including the last turn in a
session. Older workers without that marker are still handled when the next
turn for the same role begins.

## useClientEvent

Subscribe to typed client events from the avatar:
Expand Down
121 changes: 121 additions & 0 deletions packages/core/src/client-transcript.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { describe, expect, it } from 'bun:test';
import { Room, RoomEvent } from 'livekit-client';
import type { RemoteParticipant } from 'livekit-client';
import { AvatarSession } from './client';

describe('AvatarSession transcript event routing', () => {
it('keeps final-only transcript entries stable across role changes and late deltas', () => {
const session = new AvatarSession('test-session');
const room = new Room({ adaptiveStream: false, dynacast: false });
const bindRoomEvents = (session as unknown as {
bindRoomEvents: (room: Room) => void;
}).bindRoomEvents;
bindRoomEvents.call(session, room);

const transcript = session.transcript();
const updates: Array<Array<{ id: string; text: string; final: boolean }>> = [];
transcript.on('update', (entries) => {
updates.push(
entries.map(({ id, text, final }) => ({ id, text, final })),
);
});

const send = (payload: Record<string, unknown>, identity: string) => {
room.emit(
RoomEvent.DataReceived,
new TextEncoder().encode(JSON.stringify(payload)),
{ identity } as RemoteParticipant,
);
};

try {
send(
{ type: 'transcription', role: 'assistant', turn: 0, text: 'Hello' },
'worker:avatar',
);
expect(transcript.entries).toEqual([]);
expect(updates).toEqual([]);

send(
{ type: 'transcription', role: 'user', turn: 0, text: 'Hi' },
'participant:user',
);
send(
{
type: 'transcription',
role: 'user',
turn: 0,
text: '',
final: true,
},
'participant:user',
);

expect(transcript.entries).toEqual([
expect.objectContaining({
id: 'runway-transcription-user-0',
text: 'Hi',
final: true,
participantIdentity: 'participant:user',
}),
]);

send(
{
type: 'transcription',
role: 'assistant',
turn: 0,
text: '',
final: true,
},
'worker:avatar',
);
expect(transcript.entries).toEqual([
expect.objectContaining({
id: 'runway-transcription-user-0',
text: 'Hi',
final: true,
}),
expect.objectContaining({
id: 'runway-transcription-assistant-0',
text: 'Hello',
final: true,
participantIdentity: 'worker:avatar',
}),
]);

const callbackCountAfterFinals = updates.length;
send(
{
type: 'transcription',
role: 'assistant',
turn: 0,
text: '',
final: true,
},
'worker:avatar',
);
send(
{ type: 'transcription', role: 'assistant', turn: 0, text: ' late' },
'worker:avatar',
);

expect(updates).toHaveLength(callbackCountAfterFinals);
expect(transcript.entries).toEqual([
expect.objectContaining({
id: 'runway-transcription-user-0',
text: 'Hi',
final: true,
}),
expect.objectContaining({
id: 'runway-transcription-assistant-0',
text: 'Hello',
final: true,
}),
]);
} finally {
transcript.dispose();
room.removeAllListeners();
}
});
});
12 changes: 7 additions & 5 deletions packages/core/src/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -368,11 +368,13 @@ export class AvatarSession extends Emitter<AvatarEventMap> {
channel: 'custom',
});
}
this.emit(AvatarEvent.Transcript, {
...active,
final: false,
channel: 'custom',
});
if (!active.final) {
this.emit(AvatarEvent.Transcript, {
...active,
final: false,
channel: 'custom',
});
}
}
},
);
Expand Down
55 changes: 55 additions & 0 deletions packages/core/src/transcript-accumulator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,61 @@ describe('TranscriptAccumulator', () => {
expect(acc.entries[0].text).toBe('Hello');
});

it('exposes the last flat turn as final without interim mode', () => {
const acc = new TranscriptAccumulator();
const encode = (obj: object) =>
new TextEncoder().encode(JSON.stringify(obj));

acc.ingestDataChannel(
encode({ type: 'transcription', role: 'assistant', turn: 3, text: 'last answer' }),
'worker:avatar',
);
expect(acc.entries).toEqual([]);

acc.ingestDataChannel(
encode({
type: 'transcription',
role: 'assistant',
turn: 3,
text: '',
final: true,
}),
'worker:avatar',
);

expect(acc.entries).toHaveLength(1);
expect(acc.entries[0]).toMatchObject({
id: 'runway-transcription-assistant-3',
text: 'last answer',
final: true,
channel: 'custom',
});
});

it('does not let an interim update overwrite a final flat entry', () => {
const acc = new TranscriptAccumulator({ interim: true });
const encode = (obj: object) =>
new TextEncoder().encode(JSON.stringify(obj));

acc.ingestDataChannel(
encode({
type: 'transcription',
role: 'assistant',
turn: 0,
text: 'done',
final: true,
}),
'worker:avatar',
);
acc.ingestDataChannel(
encode({ type: 'transcription', role: 'assistant', turn: 0, text: ' late' }),
'worker:avatar',
);

expect(acc.entries).toHaveLength(1);
expect(acc.entries[0]).toMatchObject({ text: 'done', final: true });
});

it('dispose clears state and removes listeners', () => {
const acc = new TranscriptAccumulator();
const handler = mock();
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/transcript-accumulator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ export class TranscriptAccumulator extends Emitter<TranscriptEvents> {
changed = true;
}

if (this.interim) {
if (this.interim && !active.final) {
this.map.set(active.id, { ...active, final: false, channel: 'custom' });
changed = true;
}
Expand Down
87 changes: 87 additions & 0 deletions packages/core/src/utils/flatDeltaAccumulator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,93 @@ describe('FlatDeltaAccumulator', () => {
});
});

it('finalizes the active turn immediately on an explicit completion marker', () => {
const acc = new FlatDeltaAccumulator();
acc.ingest({ role: 'assistant', turn: 0, textDelta: 'hello ' }, 'worker:1');

const emission = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: 'there', final: true },
'worker:1',
);

expect(emission.finalized).toEqual([
{
id: 'runway-transcription-assistant-0',
text: 'hello there',
participantIdentity: 'worker:1',
},
]);
expect(emission.active).toMatchObject({
id: 'runway-transcription-assistant-0',
text: 'hello there',
final: true,
});
});

it('does not create a visible empty entry for an unobserved completion marker', () => {
const acc = new FlatDeltaAccumulator();
const emission = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: '', final: true },
'worker:1',
);

expect(emission.finalized).toEqual([]);
expect(emission.active).toMatchObject({ text: '', final: true });

const late = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: 'late' },
'worker:1',
);
expect(late.finalized).toEqual([]);
expect(late.active).toMatchObject({ text: '', final: true });
});

it('does not reopen a finalized turn when a late delta arrives', () => {
const acc = new FlatDeltaAccumulator();
acc.ingest({ role: 'assistant', turn: 0, textDelta: 'done', final: true }, 'worker:1');

const late = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: ' late' },
'worker:1',
);
expect(late.finalized).toEqual([]);
expect(late.active).toMatchObject({ text: 'done', final: true });
});

it('does not infer finality for a newer turn from an out-of-order older delta', () => {
const acc = new FlatDeltaAccumulator();
acc.ingest({ role: 'assistant', turn: 2, textDelta: 'newer' }, 'worker:1');

const older = acc.ingest(
{ role: 'assistant', turn: 1, textDelta: 'older' },
'worker:1',
);
expect(older.finalized).toEqual([]);
});

it('finalizes roles independently when explicit completion markers arrive', () => {
const acc = new FlatDeltaAccumulator();
acc.ingest({ role: 'assistant', turn: 0, textDelta: 'answer' }, 'worker:1');
acc.ingest({ role: 'user', turn: 0, textDelta: 'question' }, 'participant:me');

const assistant = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: '', final: true },
'worker:1',
);
const duplicateAssistant = acc.ingest(
{ role: 'assistant', turn: 0, textDelta: '', final: true },
'worker:1',
);
const user = acc.ingest(
{ role: 'user', turn: 0, textDelta: '', final: true },
'participant:me',
);

expect(assistant.finalized.map((entry) => entry.text)).toEqual(['answer']);
expect(duplicateAssistant.finalized).toEqual([]);
expect(user.finalized.map((entry) => entry.text)).toEqual(['question']);
});

it('emits a finalized turn exactly once across subsequent deltas', () => {
const acc = new FlatDeltaAccumulator();

Expand Down
Loading
Loading