docs(design): imported transcripts (v6 proposal) + parser prototype

src/lib/import/docx.ts

@@ -0,0 +1,11 @@
+import { documentXml } from './docxZip';
+import { paragraphs } from './docxXml';
+import { fail, TranscriptImportError, type ImportedTranscript } from './types';
+
+export async function parseDocx(bytes: Uint8Array): Promise<ImportedTranscript> {
+  try { return paragraphs(await documentXml(bytes)); }
+  catch (error) {
+    if (error instanceof TranscriptImportError) throw error;
+    return fail('MALFORMED_INPUT');
+  }
+}

src/lib/import/docxXml.ts

@@ -0,0 +1,151 @@
+import { collector, fail, IMPORT_LIMITS as L, labelled, type ImportedTranscript } from './types';
+
+const WORD = new Set(['http://schemas.openxmlformats.org/wordprocessingml/2006/main', 'http://purl.oclc.org/ooxml/wordprocessingml/main']);
+const NAME = '[A-Za-z_][A-Za-z0-9_.-]*(?::[A-Za-z_][A-Za-z0-9_.-]*)?';
+const OPEN = new RegExp(`^<(${NAME})`);
+const ATTR = new RegExp(`\\s+(${NAME})\\s*=\\s*("[^"<]*"|'[^'<]*')`, 'y');
+const CLOSE = new RegExp(`^</(${NAME})\\s*>$`);
+
+function entities(text: string): string {
+  if (/&(?!amp;|lt;|gt;|quot;|apos;|#\d+;|#x[\da-fA-F]+;)/.test(text)) fail('MALFORMED_INPUT');
+  return text.replace(/&([^;]+);/g, (_, key: string) => {
+    const builtin: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" };
+    if (builtin[key]) return builtin[key];
+    const code = key.startsWith('#x') ? parseInt(key.slice(2), 16) : Number(key.slice(1));
+    if (!Number.isInteger(code) || !(code === 9 || code === 10 || code === 13 || (code >= 32 && code <= 0xd7ff)
+      || (code >= 0xe000 && code <= 0xfffd) || (code >= 0x10000 && code <= 0x10ffff))) fail('MALFORMED_INPUT');
+    return String.fromCodePoint(code);
+  });
+}
+
+/** Bounded XML subset, not an HTML parser or a general-purpose OOXML renderer. */
+export function paragraphs(xml: string): ImportedTranscript {
+  xml = xml.replace(/^\uFEFF/, '').replace(/\r\n?/g, '\n');
+  if (/[\x00-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff]/.test(xml)) fail('MALFORMED_INPUT');
+  const out = collector();
+  out.warnings.add('DOCX_BODY_PARAGRAPHS_ONLY');
+  const stack: { name: string; local: string; word: boolean; ns: Map<string, string> }[] = [];
+  let position = 0;
+  let nodes = 0;
+  let roots = 0;
+  let bodies = 0;
+  let paragraph: string[] | null = null;
+  let paragraphBytes = 0;
+  let paragraphCount = 0;
+  function append(text: string) {
+    if (paragraph === null) fail('MALFORMED_INPUT');
+    paragraphBytes += new TextEncoder().encode(text).length;
+    if (paragraphBytes > L.turnBytes) fail('TEXT_TOO_LARGE');
+    paragraph.push(text);
+  }
+  function close() {
+    const node = stack.pop()!;
+    if (node.word && node.local === 'p') {
+      const text = paragraph!.join('');
+      if (text.trim()) {
+        const turn = labelled(text);
+        if (turn.speaker !== 'Unknown') out.warnings.add('SPEAKER_LABELS_INFERRED');
+        out.add(turn);
+      } else out.warnings.add('EMPTY_PARAGRAPHS_IGNORED');
+      paragraph = null;
+    }
+  }
+  while (position < xml.length) {
+    if (xml[position] !== '<') {
+      const next = xml.indexOf('<', position);
+      const end = next < 0 ? xml.length : next;
+      const source = xml.slice(position, end);
+      if (source.includes(']]>')) fail('MALFORMED_INPUT');
+      const text = entities(source);
+      const parent = stack[stack.length - 1];
+      if (parent?.word && parent.local === 't') append(text);
+      else if (text.trim()) fail('UNSUPPORTED_FORMAT');
+      position = end;
+      continue;
+    }
+    if (xml.startsWith('<!--', position)) {
+      const end = xml.indexOf('-->', position + 4);
+      if (end < 0 || xml.slice(position + 4, end).includes('--') || xml[end - 1] === '-') fail('MALFORMED_INPUT');
+      position = end + 3;
+      continue;
+    }
+    if (xml.startsWith('<?xml ', position) && position === 0) {
+      const end = xml.indexOf('?>', position);
+      const declaration = xml.slice(position, end + 2);
+      if (end < 0 || !/^<\?xml\s+version=(?:"1\.0"|'1\.0')(?:\s+encoding=(?:"UTF-8"|'UTF-8'))?(?:\s+standalone=(?:"(?:yes|no)"|'(?:yes|no)'))?\s*\?>$/i.test(declaration)) fail('UNSUPPORTED_FORMAT');
+      position = end + 2;
+      continue;
+    }
+    // No DTDs, entity definitions, CDATA, or processing instructions.
+    if (xml.startsWith('<!', position) || xml.startsWith('<?', position)) fail('UNSUPPORTED_FORMAT');
+    const token = /^<(?:[^<>"']|"[^"<]*"|'[^'<]*')*>/.exec(xml.slice(position))?.[0];
+    if (!token) fail('MALFORMED_INPUT');
+    position += token.length;
+    if (++nodes > L.xmlNodes) fail('ARCHIVE_LIMIT');
+    if (token.startsWith('</')) {
+      const name = CLOSE.exec(token)?.[1];
+      if (!name || stack[stack.length - 1]?.name !== name) fail('MALFORMED_INPUT');
+      close();
+      continue;
+    }
+    const match = OPEN.exec(token);
+    if (!match) fail('MALFORMED_INPUT');
+    let cursor = match[0].length;
+    const attrs = new Map<string, string>();
+    while (cursor < token.length) {
+      ATTR.lastIndex = cursor;
+      const attr = ATTR.exec(token);
+      if (!attr) break;
+      if (attrs.has(attr[1])) fail('MALFORMED_INPUT');
+      attrs.set(attr[1], entities(attr[2].slice(1, -1)));
+      cursor = ATTR.lastIndex;
+    }
+    if (!/^\s*\/?>$/.test(token.slice(cursor))) fail('MALFORMED_INPUT');
+    const ns = new Map(stack[stack.length - 1]?.ns ?? [['xml', 'http://www.w3.org/XML/1998/namespace']]);
+    for (const [key, value] of attrs) {
+      if (key === 'xmlns') ns.set('', value);
+      else if (key.startsWith('xmlns:')) {
+        if (key === 'xmlns:xmlns' || !value || (key === 'xmlns:xml' && value !== 'http://www.w3.org/XML/1998/namespace')) fail('MALFORMED_INPUT');
+        ns.set(key.slice(6), value);
+      }
+    }
+    const name = match[1];
+    const [prefix, local] = name.includes(':') ? name.split(':') : ['', name];
+    if (prefix && !ns.has(prefix)) fail('MALFORMED_INPUT');
+    for (const key of attrs.keys()) {
+      if (key.includes(':') && !key.startsWith('xmlns:') && !ns.has(key.split(':')[0])) fail('MALFORMED_INPUT');
+    }
+    const expandedAttributes = new Set<string>();
+    for (const key of attrs.keys()) {
+      if (key === 'xmlns' || key.startsWith('xmlns:')) continue;
+      const [attributePrefix, attributeLocal] = key.includes(':') ? key.split(':') : ['', key];
+      const expanded = `${attributePrefix ? ns.get(attributePrefix) : ''}#${attributeLocal}`;
+      if (expandedAttributes.has(expanded)) fail('MALFORMED_INPUT');
+      expandedAttributes.add(expanded);
+    }
+    const word = WORD.has(ns.get(prefix) ?? '');
+    if (!word) fail('UNSUPPORTED_FORMAT');
+    if (!stack.length) {
+      if (++roots !== 1 || !word || local !== 'document') fail('MALFORMED_INPUT');
+    }
+    if (word && local === 'body') {
+      if (++bodies !== 1 || stack.length !== 1) fail('MALFORMED_INPUT');
+    }
+    if (word && ['altChunk', 'object', 'drawing', 'pict', 'txbxContent', 'ins', 'del', 'instrText', 'fldSimple', 'fldChar'].includes(local)) fail('UNSUPPORTED_FORMAT');
+    if (word && local === 'p') {
+      if (paragraph !== null || !stack.some(node => node.word && node.local === 'body')) fail('MALFORMED_INPUT');
+      if (++paragraphCount > L.turns) fail('TOO_MANY_TURNS');
+      paragraph = [];
+      paragraphBytes = 0;
+    }
+    if (stack.some(node => node.word && node.local === 't')) fail('MALFORMED_INPUT');
+    if (word && local === 't' && paragraph === null) fail('MALFORMED_INPUT');
+    if (word && (local === 'tab' || local === 'br' || local === 'cr')) append(local === 'tab' ? '\t' : '\n');
+    if (word && ['sym', 'noBreakHyphen', 'softHyphen'].includes(local)) fail('UNSUPPORTED_FORMAT');
+    stack.push({ name, local, word, ns });
+    if (stack.length > L.xmlDepth) fail('ARCHIVE_LIMIT');
+    if (token.endsWith('/>')) close();
+  }
+  if (stack.length || roots !== 1 || bodies !== 1) fail('MALFORMED_INPUT');
+  return out.finish();
+}

src/lib/import/docxZip.ts

@@ -0,0 +1,138 @@
+import { crc32 } from '../export/zipStream';
+import { fail, IMPORT_LIMITS as L, TranscriptImportError, utf8 } from './types';
+
+type Entry = { name: string; start: number; compressed: number; expanded: number; crc: number; method: number };
+
+/** Small, deliberately strict ZIP32 reader. Never inflate unrelated members. */
+function directory(bytes: Uint8Array): Entry {
+  if (bytes.length > L.fileBytes) fail('FILE_TOO_LARGE');
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  function range(at: number, size: number) {
+    if (at < 0 || at + size > bytes.length) fail('MALFORMED_INPUT');
+  }
+  function u16(at: number) { range(at, 2); return view.getUint16(at, true); }
+  function u32(at: number) { range(at, 4); return view.getUint32(at, true); }
+  function extras(at: number, size: number) {
+    const end = at + size;
+    range(at, size);
+    while (at < end) {
+      if (at + 4 > end) fail('MALFORMED_INPUT');
+      const id = u16(at);
+      const length = u16(at + 2);
+      // ZIP64 and path overrides introduce a second interpretation of identity.
+      if (id === 1 || id === 0x7075) fail('UNSUPPORTED_FORMAT');
+      at += 4 + length;
+      if (at > end) fail('MALFORMED_INPUT');
+    }
+  }
+  let end = -1;
+  for (let at = bytes.length - 22; at >= Math.max(0, bytes.length - 65557); at--) {
+    if (u32(at) === 0x06054b50 && at + 22 + u16(at + 20) === bytes.length) { end = at; break; }
+  }
+  if (end < 0) fail('MALFORMED_INPUT');
+  if (u16(end + 4) || u16(end + 6) || u16(end + 8) !== u16(end + 10)) fail('UNSUPPORTED_FORMAT');
+  const count = u16(end + 10);
+  if (count > L.archiveEntries) fail('ARCHIVE_LIMIT');
+  const central = u32(end + 16);
+  if (central + u32(end + 12) !== end) fail('MALFORMED_INPUT');
+  let at = central;
+  let total = 0;
+  const names = new Set<string>();
+  const ranges: [number, number][] = [];
+  let document: Entry | undefined;
+  for (let index = 0; index < count; index++) {
+    range(at, 46);
+    if (u32(at) !== 0x02014b50 || u16(at + 34)) fail('MALFORMED_INPUT');
+    const flags = u16(at + 8);
+    const method = u16(at + 10);
+    if ((flags & ~0x080e) || (method !== 0 && method !== 8)) fail('UNSUPPORTED_FORMAT');
+    const compressed = u32(at + 20);
+    const expanded = u32(at + 24);
+    const nameLength = u16(at + 28);
+    const extraLength = u16(at + 30);
+    const next = at + 46 + nameLength + extraLength + u16(at + 32);
+    if (next > end) fail('MALFORMED_INPUT');
+    const name = utf8(bytes.subarray(at + 46, at + 46 + nameLength));
+    if (!name || /[\\\x00-\x1f\x7f]/.test(name) || name.startsWith('/') || name.includes(':')
+      || name.split('/').some(part => part === '..' || part === '.') || names.has(name)) fail('UNSAFE_ARCHIVE');
+    if (/\.(?:zip|docx|docm|xlsx|pptx|jar|7z|gz)$/i.test(name) || /(?:^|\/)embeddings\//i.test(name)
+      || /vbaProject\.bin$/i.test(name) || ((u32(at + 38) >>> 16) & 0xf000) === 0xa000) fail('UNSAFE_ARCHIVE');
+    names.add(name);
+    total += expanded;
+    if (total > L.archiveExpandedBytes || expanded > L.archiveExpandedBytes
+      || expanded > Math.max(1, compressed) * L.expansionRatio) fail('ARCHIVE_LIMIT');
+    extras(at + 46 + nameLength, extraLength);
+    const local = u32(at + 42);
+    if (u32(local) !== 0x04034b50 || u16(local + 6) !== flags || u16(local + 8) !== method) fail('MALFORMED_INPUT');
+    const localNameLength = u16(local + 26);
+    const localExtraLength = u16(local + 28);
+    range(local + 30, localNameLength + localExtraLength);
+    if (utf8(bytes.subarray(local + 30, local + 30 + localNameLength)) !== name) fail('MALFORMED_INPUT');
+    extras(local + 30 + localNameLength, localExtraLength);
+    const start = local + 30 + localNameLength + localExtraLength;
+    let localEnd = start + compressed;
+    if (localEnd > central) fail('MALFORMED_INPUT');
+    const crc = u32(at + 16);
+    if (flags & 8) {
+      const descriptor = localEnd + (u32(localEnd) === 0x08074b50 ? 4 : 0);
+      if (u32(descriptor) !== crc || u32(descriptor + 4) !== compressed || u32(descriptor + 8) !== expanded) fail('MALFORMED_INPUT');
+      localEnd = descriptor + 12;
+    } else if (u32(local + 14) !== crc || u32(local + 18) !== compressed || u32(local + 22) !== expanded) fail('MALFORMED_INPUT');
+    if (localEnd > central || (method === 0 && compressed !== expanded)) fail('MALFORMED_INPUT');
+    ranges.push([local, localEnd]);
+    if (name === 'word/document.xml') document = { name, start, compressed, expanded, method, crc };
+    at = next;
+  }
+  if (at !== end) fail('MALFORMED_INPUT');
+  ranges.sort((a, b) => a[0] - b[0]);
+  let previousEnd = 0;
+  for (const [start, stop] of ranges) {
+    if (start !== previousEnd) fail('UNSAFE_ARCHIVE'); // overlaps, hidden local entries, executable preambles
+    previousEnd = stop;
+  }
+  if (previousEnd !== central) fail('UNSAFE_ARCHIVE');
+  if (!document) fail('MALFORMED_INPUT');
+  if (document.expanded > L.documentBytes) fail('ARCHIVE_LIMIT');
+  return document;
+}
+
+export async function documentXml(bytes: Uint8Array): Promise<string> {
+  const entry = directory(bytes);
+  const compressed = bytes.subarray(entry.start, entry.start + entry.compressed);
+  let output: Uint8Array;
+  if (entry.method === 0) output = compressed;
+  else {
+    // Small compressed chunks bound expansion between backpressure checks; never
+    // hand an entire attacker-controlled compressed member to the inflater at once.
+    let compressedOffset = 0;
+    const source = new ReadableStream<BufferSource>({
+      pull(controller) {
+        if (compressedOffset === compressed.length) { controller.close(); return; }
+        const next = Math.min(compressedOffset + 512, compressed.length);
+        controller.enqueue(new Uint8Array(compressed.subarray(compressedOffset, next)));
+        compressedOffset = next;
+      },
+    });
+    const reader = source.pipeThrough(new DecompressionStream('deflate-raw')).getReader();
+    const chunks: Uint8Array[] = [];
+    let length = 0;
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        length += value.length;
+        if (length > L.documentBytes || length > entry.expanded || length > Math.max(1, entry.compressed) * L.expansionRatio) fail('ARCHIVE_LIMIT');
+        chunks.push(value);
+      }
+      output = new Uint8Array(length);
+      let offset = 0;
+      for (const chunk of chunks) { output.set(chunk, offset); offset += chunk.length; }
+    } catch (error) {
+      await reader.cancel().catch(() => undefined);
+      if (error instanceof TranscriptImportError) throw error;
+      return fail('MALFORMED_INPUT');
+    } finally { reader.releaseLock(); }
+  }
+  if (output.length !== entry.expanded || crc32(output) !== entry.crc) fail('MALFORMED_INPUT');
+  return utf8(output);
+}

src/lib/import/index.ts

@@ -0,0 +1,4 @@
+export { parseWebVtt, parseSrt } from './subtitles';
+export { parseDocx } from './docx';
+export { IMPORT_LIMITS, TranscriptImportError } from './types';
+export type { ImportedTranscript, ImportedTurn, ImportErrorCode } from './types';

src/lib/import/subtitles.ts

@@ -0,0 +1,56 @@
+import { collector, fail, inputText, labelled, type ImportedTranscript } from './types';
+
+function timestamp(value: string, srt: boolean): number {
+  const pattern = srt ? /^(\d{2,}):(\d{2}):(\d{2}),(\d{3})$/ : /^(?:(\d{2,}):)?(\d{2}):(\d{2})\.(\d{3})$/;
+  const match = pattern.exec(value);
+  if (!match || Number(match[2]) > 59 || Number(match[3]) > 59) fail('MALFORMED_INPUT');
+  const result = ((Number(match[1] ?? 0) * 60 + Number(match[2])) * 60 + Number(match[3])) * 1000 + Number(match[4]);
+  if (!Number.isSafeInteger(result)) fail('MALFORMED_INPUT');
+  return result;
+}
+
+function parse(input: string | Uint8Array, srt: boolean): ImportedTranscript {
+  const text = inputText(input);
+  const blocks = text.replace(/\n+$/, '').split(/\n[ \t]*\n/);
+  const out = collector();
+  if (!srt) {
+    const header = blocks.shift()?.split('\n') ?? [];
+    if (!/^WEBVTT(?:[ \t].*)?$/.test(header[0] ?? '') || header.some(line => line.includes('-->'))) fail('MALFORMED_INPUT');
+    if (header.length > 1) out.warnings.add('VTT_METADATA_IGNORED');
+  }
+  for (const block of blocks) {
+    if (!block.trim()) continue;
+    const lines = block.split('\n');
+    if (!srt && /^(?:NOTE(?:[ \t]|$)|STYLE$|REGION$)/.test(lines[0])) {
+      out.warnings.add('VTT_METADATA_IGNORED');
+      continue;
+    }
+    if (srt) {
+      if (!/^\d+$/.test(lines.shift() ?? '')) fail('MALFORMED_INPUT');
+    } else if (!lines[0].includes('-->')) lines.shift(); // optional cue identifier
+    const timing = /^(\S+) --> (\S+)(?:[ \t]+(.+))?$/.exec(lines.shift() ?? '');
+    if (!timing || (srt && timing[3])) fail('MALFORMED_INPUT');
+    if (timing[3]) out.warnings.add('VTT_SETTINGS_IGNORED');
+    const startMs = timestamp(timing[1], srt);
+    const endMs = timestamp(timing[2], srt);
+    if (endMs <= startMs) fail('MALFORMED_INPUT');
+    const payload = lines.join('\n');
+    // Multiple voices per cue are ambiguous: refuse rather than attribute to one.
+    const voice = !srt && /^<v(?:\.[\w-]+)*[ \t]+([^<>\n]+)>([\s\S]*?)(?:<\/v>)?$/.exec(payload);
+    if (!srt && /<\/?v(?:[\s.>])/.test(payload) && (!voice || /<\/?v(?:[\s.>])/.test(voice[2]))) fail('MALFORMED_INPUT');
+    const turn = voice ? { speaker: voice[1].trim(), text: voice[2] } : labelled(payload);
+    // Preserve all other markup literally, including injection-looking strings.
+    // Decode only the six WebVTT escapes, after recognizing voice syntax.
+    if (!srt) {
+      const escapes: Record<string, string> = { amp: '&', lt: '<', gt: '>', nbsp: '\u00a0', lrm: '\u200e', rlm: '\u200f' };
+      turn.text = turn.text.replace(/&(amp|lt|gt|nbsp|lrm|rlm);/g, (_, key: string) => escapes[key]);
+      turn.speaker = turn.speaker.replace(/&(amp|lt|gt|nbsp|lrm|rlm);/g, (_, key: string) => escapes[key]);
+    }
+    if (/<[^>]+>/.test(turn.text)) out.warnings.add('LITERAL_MARKUP_PRESERVED');
+    if (!voice && turn.speaker !== 'Unknown') out.warnings.add('SPEAKER_LABELS_INFERRED');
+    out.add({ ...turn, startMs, endMs });
+  }
+  return out.finish();
+}
+export function parseWebVtt(input: string | Uint8Array): ImportedTranscript { return parse(input, false); }
+export function parseSrt(input: string | Uint8Array): ImportedTranscript { return parse(input, true); }