From 30cdedd0fa0f2893518dcce102632b50d4307285 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 13 Sep 2026 08:05:56 -0600 Subject: [PATCH 1/2] webui: split TTS text on sentences in Latin-script text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit splitTtsChunks treats 。！？!?；;… as sentence terminators. ASCII '.' is not among them, so a paragraph of English prose is one unsplittable sentence and falls through to the fixed-width cut, which lands mid-word: splitTtsChunks("The first sentence is here. The second follows it closely. A third arrives now. ...", 60) [60] The first sentence is here. The second follows it closely. A [60] third arrives now. And a fourth, rather longer than the oth [60] ers, continues past the point where a small budget would hav [26] e to cut. Finally a fifth. Each chunk is a separate synthesis request, so a word split across two of them is pronounced as two fragments. '.' now terminates a sentence, with the guards that make it ambiguous in the first place: not between digits, not after an abbreviation or a single-letter initial, and only before whitespace. So 3.14159, Dr. Smith, J. R. R. Tolkien, file.txt and example.com stay whole. Where no sentence boundary fits the budget, the fallback breaks on words rather than characters, so only a token longer than the entire budget is cut mid-word. The same text now gives: [58] The first sentence is here. The second follows it closely. [20] A third arrives now. [59] And a fourth, rather longer than the others, continues past [49] the point where a small budget would have to cut. [16] Finally a fifth. Two smaller fixes alongside: a "Speaker 1:" prefix is counted against the budget, since it is repeated onto every chunk a line produces and those chunks otherwise exceed the caller's limit; and chunks are trimmed, so a prefixed chunk no longer carries a double space. CJK behaviour is unchanged -- the existing terminators still apply, and '.' is additive. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01EkxqpYvUbjCpRDnFiNiVfx --- webui/native/dist/index.html | 18 ++--- webui/native/src/lib/text.ts | 123 +++++++++++++++++++++++++++++++---- 2 files changed, 120 insertions(+), 21 deletions(-) diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index cb56598b7..e70737bb9 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
diff --git a/webui/native/src/lib/text.ts b/webui/native/src/lib/text.ts index 038a715a0..08acc34ff 100644 --- a/webui/native/src/lib/text.ts +++ b/webui/native/src/lib/text.ts @@ -1,32 +1,131 @@ const speakerLine = /^\s*(Speaker\s+\d+\s*:)\s*(.*)$/i; -const sentenceParts = /[^。！？!?；;…]*[。！？!?；;…]+|[^。！？!?；;…]+$/g; + +// Sentence terminators. '.' is handled separately by endsSentence, because it is +// the only one that is ambiguous: it also marks decimals, abbreviations, +// initials and file extensions. +const terminators = new Set(['。', '！', '？', '!', '?', '；', ';', '…', '.']); + +// Abbreviations that end in a full stop without ending a sentence. Not +// exhaustive -- it cannot be -- but it covers what prose actually contains, and +// a miss costs a split in a slightly wrong place, not a failure. +const abbreviations = new Set([ + 'mr', 'mrs', 'ms', 'dr', 'prof', 'sr', 'jr', 'st', 'mt', 'rev', 'hon', + 'vs', 'etc', 'eg', 'ie', 'approx', 'dept', 'est', 'fig', 'no', 'vol', + 'jan', 'feb', 'mar', 'apr', 'jun', 'jul', 'aug', 'sep', 'sept', 'oct', + 'nov', 'dec', 'inc', 'ltd', 'co', 'corp', +]); + +function isDigit(character: string): boolean { + return character >= '0' && character <= '9'; +} + +function isWordCharacter(character: string): boolean { + return /[\p{L}\p{N}']/u.test(character); +} + +/** Whether a full stop ends a sentence rather than a number or an abbreviation. */ +function endsSentence(text: string, index: number): boolean { + // Must be followed by whitespace or end of text, which keeps file.txt and + // example.com intact. + const next = text[index + 1]; + if (next !== undefined && !/\s/.test(next)) return false; + + // Not a decimal point. + if (index > 0 && isDigit(text[index - 1]) && next !== undefined && isDigit(next)) return false; + + let start = index; + while (start > 0 && isWordCharacter(text[start - 1])) start -= 1; + const word = text.slice(start, index); + + // A single letter is an initial: "J. R. R. Tolkien". + if (word.length === 1 && /\p{L}/u.test(word)) return false; + + return !abbreviations.has(word.toLowerCase()); +} + +/** + * Break text after each sentence terminator, keeping the terminator and any + * following whitespace attached to the sentence it ends. + */ +function splitSentences(text: string): string[] { + const sentences: string[] = []; + let start = 0; + + for (let index = 0; index < text.length; index += 1) { + if (!terminators.has(text[index])) continue; + if (text[index] === '.' && !endsSentence(text, index)) continue; + + let end = index + 1; + while (end < text.length && terminators.has(text[end])) end += 1; + while (end < text.length && /\s/.test(text[end])) end += 1; + + sentences.push(text.slice(start, end)); + start = end; + index = end - 1; + } + + if (start < text.length) sentences.push(text.slice(start)); + return sentences; +} + +/** + * Cut text into budget-sized pieces at whitespace, falling back to a hard cut + * only for a single token that is itself longer than the budget. + */ +function breakOnWords(text: string, budget: number): string[] { + const pieces: string[] = []; + let rest = text.trim(); + + while (rest.length > budget) { + let cut = rest.lastIndexOf(' ', budget); + if (cut <= 0) cut = budget; + + const piece = rest.slice(0, cut).trim(); + if (piece) pieces.push(piece); + rest = rest.slice(cut).trim(); + } + + if (rest) pieces.push(rest); + return pieces; +} function splitLongLine(line: string, budget: number): string[] { const match = speakerLine.exec(line); const prefix = match ? `${match[1]} ` : ''; const body = match ? match[2] : line.trim(); - const sentences = body.match(sentenceParts) || [body]; + + // The prefix is repeated onto every chunk, so it has to come out of the + // budget or chunks carrying one exceed the limit the caller asked for. + const room = Math.max(1, budget - prefix.length); + + const sentences = splitSentences(body); + if (!sentences.length) sentences.push(body); + const chunks: string[] = []; let current = ''; - for (const sentence of sentences) { - if (current && current.length + sentence.length > budget) { - chunks.push(prefix + current); + for (const raw of sentences) { + const sentence = raw.trimEnd(); + if (!sentence) continue; + + if (current && current.length + 1 + sentence.length > room) { + chunks.push(prefix + current.trim()); current = ''; } - if (sentence.length <= budget) { - current += sentence; + if (sentence.length <= room) { + current = current ? `${current} ${sentence}` : sentence; continue; } if (current) { - chunks.push(prefix + current); + chunks.push(prefix + current.trim()); current = ''; } - for (let offset = 0; offset < sentence.length; offset += budget) { - chunks.push(prefix + sentence.slice(offset, offset + budget)); - } + // No sentence boundary fits, so fall back to word boundaries. Only a token + // longer than the whole budget is ever cut mid-word. + for (const piece of breakOnWords(sentence, room)) chunks.push(prefix + piece); } - if (current) chunks.push(prefix + current); + if (current) chunks.push(prefix + current.trim()); + return chunks.length ? chunks : [line]; } From 335220ac459aa23b8ecaf2ab51be90673f28df79 Mon Sep 17 00:00:00 2001 From: Chris Thompson Date: Sun, 13 Sep 2026 19:42:37 -0600 Subject: [PATCH 2/2] webui: keep the spacing that separated the sentences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Packing threw away the whitespace splitSentences had carefully kept and put a single space back in its place. Three coupled lines assumed that separator was always one space: trimEnd() dropped it, the budget check added 1 for it, and the join wrote it. Sentences in Chinese, Japanese and Korean are adjacent -- a full-width terminator and nothing else -- so this invented a space that was never in the text. That is not only an extra request: it changes what the model is asked to speak, and it spends a character of the budget, so three 20-character sentences stopped fitting in two 40-character chunks. before: [20] ...吧。 [20] ...吧。 [20] ...吧。 after: [40] ...吧。...吧。 [20] ...吧。 The separator that actually followed each sentence is carried instead. One other case changes with it, deliberately: "First one. Second one." keeps its four spaces rather than being silently collapsed to one. Collapsing was an edit to the user's text that nobody asked the chunker to make. Checked against a 17-case corpus covering abbreviations, initials, speaker prefixes, over-long tokens, multiple spacing, newlines, CJK and mixed scripts; those two cases are the only ones whose output moves. --- webui/native/dist/index.html | 16 ++++++++-------- webui/native/src/lib/text.ts | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index e70737bb9..6902014dd 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@
diff --git a/webui/native/src/lib/text.ts b/webui/native/src/lib/text.ts index 08acc34ff..8bf711f5d 100644 --- a/webui/native/src/lib/text.ts +++ b/webui/native/src/lib/text.ts @@ -103,22 +103,34 @@ function splitLongLine(line: string, budget: number): string[] { const chunks: string[] = []; let current = ''; + // Whatever whitespace actually followed the last sentence added to `current`. + // splitSentences keeps it, and it has to be carried rather than replaced with + // a space: sentences in Chinese, Japanese and Korean are adjacent, separated + // by a full-width terminator and nothing else. Inserting a space there both + // changes the text the model is asked to speak and spends a character of the + // budget, so three 20-character sentences stop fitting in two 40-character + // chunks and become three requests instead of two. + let separator = ''; for (const raw of sentences) { const sentence = raw.trimEnd(); if (!sentence) continue; + const trailing = raw.slice(sentence.length); - if (current && current.length + 1 + sentence.length > room) { + if (current && current.length + separator.length + sentence.length > room) { chunks.push(prefix + current.trim()); current = ''; + separator = ''; } if (sentence.length <= room) { - current = current ? `${current} ${sentence}` : sentence; + current = current ? `${current}${separator}${sentence}` : sentence; + separator = trailing; continue; } if (current) { chunks.push(prefix + current.trim()); current = ''; + separator = ''; } // No sentence boundary fits, so fall back to word boundaries. Only a token // longer than the whole budget is ever cut mid-word.