diff --git a/.changeset/nodes-prop.md b/.changeset/nodes-prop.md new file mode 100644 index 0000000..136e865 --- /dev/null +++ b/.changeset/nodes-prop.md @@ -0,0 +1,5 @@ +--- +'@fuzdev/mdz': minor +--- + +perf: add an optional `nodes` prop to `Mdz` for rendering a pre-parsed `MdzNode` tree, skips `mdz_parse` \ No newline at end of file diff --git a/.gitignore b/.gitignore index e0cbe20..f6ced5f 100644 --- a/.gitignore +++ b/.gitignore @@ -40,8 +40,9 @@ tmp/ # Log files *.log -# Benchmarks (local-only baseline) +# Benchmarks (local-only baselines) src/benchmarks/baseline.json +src/benchmarks/render/baseline.json # Workflow /worktree diff --git a/CLAUDE.md b/CLAUDE.md index 017d3f7..8c6338a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -39,22 +39,42 @@ server. ## Benchmarks ```bash -npm run benchmark # run parser benchmarks, compare against baseline -npm run benchmark:save # save current results as the new baseline -npm run benchmark:clean # remove the local baseline (forces a fresh seed) +npm run benchmark # run parser benchmarks, compare against baseline +npm run benchmark:save # save current results as the new baseline +npm run benchmark:clean # remove the local baseline (forces a fresh seed) +npm run benchmark:render # run render benchmarks (SSR + MdzStreamState) +npm run benchmark:render:save # save render results as the new baseline +npm run benchmark:render:clean # remove the local render baseline ``` -The baseline lives at `src/benchmarks/baseline.json` and is **gitignored** — -local-only. - -The suite compares the sync parser (`lexer-based`) against the streaming -pipeline under several feed strategies: `streaming` / `opcodes-only` -(one-shot feed, with and without the tree bridge), `streaming 64B` (64-byte -chunks — exercises cross-chunk buffer compaction and search-memo -invalidation), and `char-by-char` (small inputs only — large inputs would hit -the known O(n²) buffer growth in `feed()` and stall the run). The "large -dense inline" input guards against quadratic rescans on delimiter-dense -single-paragraph docs. +The baselines live at `src/benchmarks/baseline.json` and +`src/benchmarks/render/baseline.json` and are **gitignored** — local-only. +Inputs are shared between the two suites via `benchmark_inputs.ts`. + +The parser suite (`mdz.benchmark.ts`, run via `gro run`) compares the sync +parser (`lexer-based`) against the streaming pipeline under several feed +strategies: `streaming` / `opcodes-only` (one-shot feed, with and without the +tree bridge), `streaming 64B` (64-byte chunks — exercises cross-chunk buffer +compaction and search-memo invalidation), and `char-by-char` (inputs up to +30KB — linear on line-bounded input at ~4 MB/s, per-feed overhead dominates; +the cap just keeps the suite's wall clock reasonable). Adversarial inputs +guard specific non-linear failure modes: "large dense inline" (failed +delimiter/closing-tag searches must not rescan — the sync `#index_of` memo), +"mismatched tags" (`try_close_tag`'s `open_tag_counts` O(1) bail), "hold +line code" / "hold line link" (held candidates re-entered every feed must +resume their terminator scans via the `buffer_index_of` memo, not rescan the +growing line), and "nested inline cap" (deep `[`/`` past the inline nesting +cap — the sync lexer's revert re-scan and the streaming `open_link_tag_depth` +cap must keep both parsers linear and overflow-free). A residual super-linear term remains when a hold pins the +buffer against compaction (V8 rope flattening per feed) — only a chunked +buffer abstraction would remove it, and it needs a multi-KB single line fed +in small chunks to matter. + +The render suite (`render.benchmark.test.ts`) runs under vitest because the +Svelte pipeline needs the compiler — it measures SSR through `Mdz` and +`MdzStream` (via `svelte/server`'s `render`, no DOM needed) and the reactive +consumer's `MdzStreamState.apply_batch` in isolation. It's gated behind the +`BENCHMARK_RENDER` env var, so plain `gro test` skips it. ## Key dependencies @@ -84,7 +104,11 @@ code render through an injection seam (see below). mdz is a **markdown dialect and renderer**: - a deliberately small, unambiguous grammar (no setext headings, no reference - links, intraword underscores stay literal, etc.) + links, intraword underscores stay literal, inline link/tag nesting capped so + deeply nested `[`/`` render literal — a strict untrusted-content bound (both + parsers agree on pure nesting; on a cap-saturating prefix of _unclosed_ tags + they diverge, adversarial-only — see `MAX_INLINE_NESTING_DEPTH`); only + pathological input reaches it) - an incremental streaming parser (`MdzStreamParser`) producing opcodes, for rendering partial/streaming input - a synchronous parser (`mdz_parse`) producing an `MdzNode` tree @@ -100,19 +124,19 @@ mdz is a **markdown dialect and renderer**: ## Syntax -| Feature | Syntax | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Inline code | `` `code` `` | -| Bold / italic / strike | `**bold**`, `_italic_`, `~~strike~~` (single `~` is literal) | -| Links | auto-detected URLs, `/internal/path`, `./relative`, `[text](url)` | -| Headings | `# Heading` (column 0; gets a slugified `id` for fragment links) | -| Lists | `- item` / `1. item` (column 0 starts; indent nests, blank lines contained, items hold paragraphs/lists/code blocks/blockquotes/tables on their own indented lines; marker-line remainder is inline-only) | -| Blockquotes | `> ` per line (no lazy continuation); nesting via `>>` or `> > `; bare `>` is the in-quote paragraph break; a blank line ends the quote; content is a mini-document | -| Code blocks | fenced with optional language hints; an unclosed fence consumes to EOF (or to the end of its enclosing blockquote) | -| Horizontal rule | `---` on its own line | +| Feature | Syntax | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Inline code | `` `code` `` | +| Bold / italic / strike | `**bold**`, `_italic_`, `~~strike~~` (single `~` is literal) | +| Links | auto-detected URLs, `/internal/path`, `./relative`, `[text](url)` | +| Headings | `# Heading` (column 0; gets a slugified `id` for fragment links) | +| Lists | `- item` / `1. item` (column 0 starts; indent nests, blank lines contained, items hold paragraphs/lists/code blocks/blockquotes/tables on their own indented lines; marker-line remainder is inline-only) | +| Blockquotes | `> ` per line (no lazy continuation); nesting via `>>` or `> > `; bare `>` is the in-quote paragraph break; a blank line ends the quote; content is a mini-document | +| Code blocks | fenced with optional language hints; an unclosed fence consumes to EOF (or to the end of its enclosing blockquote) | +| Horizontal rule | `---` on its own line | | Tables | `\| a \| b \|` rows + a `\| --- \| :-: \|` delimiter row (colons set per-column alignment); leading **and** trailing `\|` required; inline-only cells (`` `code` `` protects pipes, `\|` is a literal pipe); a header/delimiter column-count mismatch stays a paragraph; body rows pad/truncate at render | -| Components / elements | `` / `` (must be registered) | -| Paragraphs / breaks | blank line between paragraphs; single newlines are soft breaks; `
` (registered) for hard breaks | +| Components / elements | `` / `` (must be registered) | +| Paragraphs / breaks | blank line between paragraphs; single newlines are soft breaks; `
` (registered) for hard breaks | Whitespace follows standard markdown semantics: the parser preserves whitespace in text nodes (literal `\n`, no `
` nodes), but the default @@ -154,7 +178,7 @@ two pipelines agree by construction. Tables open at the top level, inside blockquotes (free via the recursive quote parser), and as list-item block children — a deeper pipe row whose delimiter is likewise indented, recognized on the fence/quote-in-item dispatch and ended by a dedent. They never start on a -marker line (`- | a |` is inline text, like `` - ```ts ``). +marker line (`- | a |` is inline text, like `- ```ts`). ### Parsing pipeline @@ -173,9 +197,15 @@ marker line (`- | a |` is inline text, like `` - ```ts ``). ### Rendering -- `Mdz.svelte` — render static content: `` +- `Mdz.svelte` — render static content: ``, or a + pre-parsed tree via `` to skip `mdz_parse` at render + (e.g. build-time-parsed docs content); pass exactly one of `content`/`nodes` - `MdzStream.svelte` — render streaming content from an `MdzStreamState` -- `MdzNodeView.svelte` / `MdzStreamNodeView.svelte` — recursive node renderers +- `MdzNodeView.svelte` / `MdzStreamNodeView.svelte` — recursive node + renderers; the recursion is **snippet-based** (a `render_node` snippet + calling itself), so a tree costs one component instance and one set of + context reads at its root rather than per node — contexts are therefore + resolved once per tree, not per subtree - `MdzRoot.svelte` — context provider for `base`, `components`, `elements`, `code`, and `codeblock` - `MdzPrecompiled.svelte` — wrapper for preprocessor output @@ -232,8 +262,20 @@ documented surface is the parse/stream/render/preprocess API plus the ## Testing -Tests live in `src/test/` (not co-located). Fixture-based tests drive both the -parser and the preprocessor: +Tests live in `src/test/` (not co-located). The runtime renderers are covered +by SSR tests (`mdz_render.test.ts` — `svelte/server`'s `render`, no DOM +environment); `mdz_to_svelte.render_parity.test.ts` binds `mdz_to_svelte`'s +generated markup to the runtime renderer's SSR output across the whole fixture +corpus (normalized), and `mdz_render.stream_parity.test.ts` binds `MdzStream`'s +SSR to `Mdz`'s — so the build-time, sync, and streaming render paths can't +drift silently. Complexity invariants are guarded two ways: +`mdz_scaling.test.ts` runs nested-construct and streaming-hold inputs at large +`n` under generous timeouts (a regression overflows or times out), and +`mdz_scaling_ratio.test.ts` asserts machine-independent work ratios via the +`mdz_debug_work` counter (2× input → ~2× work is linear, ~4× quadratic — a +ratio, so thermal drift cancels); `mdz_nesting_cap.test.ts` locks the +`MAX_INLINE_NESTING_DEPTH` boundary and the sync/stream differential battery. +Fixture-based tests drive both the parser and the preprocessor: | Category | Input | Tests | | --------------------------------- | -------------- | --------------------------- | diff --git a/package.json b/package.json index 0ed1735..5d47b98 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,10 @@ "deploy": "gro deploy", "benchmark": "gro run src/benchmarks/mdz.benchmark.ts", "benchmark:save": "gro run src/benchmarks/mdz.benchmark.ts --save", - "benchmark:clean": "rm -f src/benchmarks/baseline.json" + "benchmark:clean": "rm -f src/benchmarks/baseline.json", + "benchmark:render": "BENCHMARK_RENDER=1 gro test render.benchmark", + "benchmark:render:save": "BENCHMARK_RENDER=save gro test render.benchmark", + "benchmark:render:clean": "rm -f src/benchmarks/render/baseline.json" }, "type": "module", "engines": { diff --git a/src/benchmarks/benchmark_inputs.ts b/src/benchmarks/benchmark_inputs.ts new file mode 100644 index 0000000..bdb7a75 --- /dev/null +++ b/src/benchmarks/benchmark_inputs.ts @@ -0,0 +1,315 @@ +/** + * Shared inputs for the mdz benchmark suites — the parser suite + * (`mdz.benchmark.ts`, run via `gro run`) and the render suite + * (`render.benchmark.test.ts`, run via vitest for the Svelte pipeline). + */ + +export interface BenchmarkInput { + name: string; + content: string; +} + +/** Mixed-structure document: headings, inline formatting, links, fences. */ +export const generate_large_input = (): string => { + const sections: Array = []; + for (let i = 0; i < 50; i++) { + sections.push(`## Section ${i + 1} + +This is paragraph ${i + 1} with **bold** and _italic_ text. +Here's a \`code snippet\` and a [link](https://fuz.dev/${i}). + +\`\`\` +code block ${i + 1} +const value = ${i}; +\`\`\` + +Some more text with https://auto.link/${i} and ~~strikethrough~~ content.`); + } + return `# Large Document\n\n${sections.join('\n\n')}`; +}; + +// Blockquote-dense input: nesting depth changes, in-quote blanks, lists and +// fences inside quotes, quotes inside items, and delimiter-dense quoted +// paragraphs. Guards the nested-parser pipeline — per-line prefix matching, +// one-level stripping, and (streaming) opcode forwarding with offset +// remapping must all stay linear. +export const generate_blockquote_heavy_input = (): string => { + const sections: Array = []; + for (let i = 0; i < 40; i++) { + sections.push(`## Quote group ${i + 1} + +> **Reviewer ${i}** wrote about snake_case_idents and ~/dev/path_${i}: +> +> The earlier draft of section_${i} said something _different_ here, +> with more_snake_case and x_${i} markers throughout the paragraph. +> +> - first point about \`api_call_${i}\` +> - second point with **bold** content +> - a nested bullet under it +> +> > and a nested quote quoting reply_${i} +> > across two lines +> +> \`\`\`ts +> const value_${i} = compute(${i}); +> \`\`\` + +1. step with a quoted note + > attached to item ${i} + > continuing the quote +2. follow-up step ${i}`); + } + return `# Blockquote Heavy\n\n${sections.join('\n\n')}`; +}; + +// List-heavy input: deep nesting, blank-line containment, in-item fences, and +// delimiter-dense item content (snake_case `_` failed candidates; the `~/dev/` +// path tildes take the single-`~` literal fast path). Locks in two costs: +// per-line classification inside open lists must stay invisible, and the +// inline closer scans' run-boundary probe must stay amortized-linear on +// delimiter-dense items. +export const generate_list_heavy_input = (): string => { + const sections: Array = []; + for (let i = 0; i < 40; i++) { + sections.push(`## Step group ${i + 1} + +1. **Setup ${i}** + + Install the dependency_name_${i} package and set the env_var_${i} value + for the ~/dev/repo_${i} checkout before continuing. + + \`\`\`bash + npm install pkg_${i} + \`\`\` + +2. **Run ${i}** + - nested bullet with snake_case_idents and ~/dev/path_${i} refs + - another _nested_ bullet with more_snake_case here + - deeper level with x_${i} and y_${i} markers + - back out to the sibling level + +3. Final step ${i} with trailing prose + continuing on an indented line`); + } + return `# List Heavy\n\n${sections.join('\n\n')}`; +}; + +// Table-heavy input: many pipe tables with inline-dense cells (bold, code, +// links, strikethrough), code-span and `\|` literal pipes, and per-column +// alignment. Locks in two costs: per-row cell splitting + cell inline parsing +// must stay linear across hundreds of cells (the streaming path shares one +// lexer/parser per table via `MdzTableCellParser`), and the streaming header +// hold (a `|` row held until its delimiter line arrives) must not rescan +// quadratically — the 64B feed strategy splits rows mid-line. +export const generate_table_heavy_input = (): string => { + const sections: Array = []; + for (let i = 0; i < 40; i++) { + sections.push(`## Table group ${i + 1} + +| Name | \`Type\` | Default | Notes | +| :--- | :----: | ------: | :---- | +| field_${i}_a | \`string\` | \`""\` | a **bold** note with ./path_${i} | +| field_${i}_b | \`number\` | \`0\` | union \`A_${i} | B_${i}\` protected | +| field_${i}_c | \`boolean\` | \`false\` | escaped a \\| b literal | +| field_${i}_d | \`Array\` | \`[]\` | see [docs](/docs/${i}) and ~~old~~ | +| field_${i}_e | \`Map\` | \`null\` | trailing _italic_ ${i} |`); + } + return `# Table Heavy\n\n${sections.join('\n\n')}`; +}; + +/** The shared benchmark corpus, ordered small to pathological. */ +export const benchmark_inputs: Array = [ + {name: 'tiny', content: 'hello **bold** world'}, + { + name: 'small', + content: `# Small Document + +This is a _simple_ paragraph with **bold** text and \`inline code\`. + +Here's a link: [click here](https://fuz.dev) and an auto-link https://fuz.dev/path. + +Some ~~strikethrough~~ text and more _italic_ words.`, + }, + { + name: 'medium', + content: `# Medium Document + +## Introduction + +This is a **medium-sized** document that tests various mdz features. +It has multiple paragraphs with _italic_, **bold**, and \`code\` formatting. + +## Code Examples + +Here's some inline \`code\` and a code block: + +\`\`\`typescript +const x = 42; +function hello() { + return 'world'; +} +\`\`\` + +## Links and References + +Visit [the docs](https://docs.fuz.dev) for more info. +Also see https://fuz.dev/api and /internal/path for details. + +## Formatting + +This paragraph has **bold with _nested italic_** and standalone ~~strikethrough~~ text. +Multiple **bold** words in a **single** line with \`code\` mixed in. + +--- + +Final section after a horizontal rule. + +More content with [multiple](https://a.com) links [here](https://b.com) and [there](/c).`, + }, + {name: 'large', content: generate_large_input()}, + { + name: 'angle brackets', + // Angle brackets without closing tags — exercises tag bail-out paths. + // Before the pre-check fix, unclosed tags like caused O(n*k) + // scanning through the rest of the document. + content: `# TypeScript-Heavy Document + +### Why object literals beat Pick + +A \`Pick\` pattern forces every consumer to import the +god type. Small standalone interfaces have no such coupling. + +## Generic Signatures + +\`\`\`typescript +export interface GitDeps { + checkout: (options: {branch: string}) => Promise>; + push: (options: {cwd?: string}) => Promise>; +} + +export const update = async ( + repos: Array, + updates: Map, +): Promise => {}; + +export const read_json = (path: string): Promise => {}; +\`\`\` + +### Narrowing with \`Pick<>\` + +\`Pick<>\` on small \`*Deps\` interfaces is fine: + +\`\`\`typescript +password: Pick; +\`\`\` + +The anti-pattern is \`Pick\` — coupling every consumer to a large type. + +## More Generics + +Functions with Array, Promise, and Map in prose. + +| Type | Example | +| --- | --- | +| \`Array\` | list of names | +| \`Promise>\` | async result | +| \`Pick\` | narrowed deps |`, + }, + { + name: 'many angles', + // Many unclosed angle brackets in a single paragraph — worst case for + // repeated tag bail-outs within one parse unit + content: Array.from( + {length: 50}, + (_, i) => `Item ${i}: Array and Map> end.`, + ).join('\n'), + }, + {name: 'list heavy', content: generate_list_heavy_input()}, + {name: 'blockquote heavy', content: generate_blockquote_heavy_input()}, + {name: 'table heavy', content: generate_table_heavy_input()}, + { + name: 'large dense inline', + // Large doc dense in unclosed delimiter candidates — ``-shaped + // generics searching for closing tags that never exist. Each failed + // search must not rescan to the end of the document (the `#index_of` + // memo), or parsing goes quadratic and large real-world docs hang. + // The `~/dev` home paths used to be the other half of the pathology + // (single-`~` strikethrough scans); since strikethrough moved to GFM + // `~~`, a single `~` is literal with no scan at all, and the paths now + // exercise that fast path. + content: Array.from( + {length: 1600}, + (_, i) => + `The ~/dev/repo_${i} crate returns Vec and Box while ~/dev/other_${i} uses Option>> in its API surface. `, + ).join('\n'), + }, + { + name: 'mismatched tags', + // Many unclosed `` opens each followed by a `` closer that + // never matches — the closer walks toward the run boundary past every + // leaked frame. Guards `try_close_tag`'s `open_tag_counts` O(1) bail + // (and the sync lexer's memoized closing-tag search): without them this + // input is quadratic even in a single one-shot feed. + content: Array.from({length: 2000}, (_, i) => `x`).join(''), + }, + { + name: 'hold line code', + // One giant line: an unclosed backtick under open bold. The streaming + // parser holds at the backtick (the closer may still arrive), and every + // feed re-enters the same candidate — the closer/newline scans must be + // memoized (`buffer_index_of`) or chunked feeds go quadratic in the + // line length. Sized so the char-by-char row stays affordable. + content: '**a `' + 'x'.repeat(16_000), + }, + { + name: 'hold line link', + // One giant unterminated `[text](url…` — the streaming parser holds at + // the `]` while the reference grows, re-entering every feed; the `)` + // terminator scan must be memoized or chunked feeds go quadratic. + content: '[text](https://example.com/' + 'x'.repeat(16_000), + }, + { + name: 'dense inline code', + // mdz's TSDoc/API-docs shipping workload: many short `` `identifier` `` + // spans per line across many lines. The rest of the corpus has no + // backtick-dense input, yet the streaming `try_code` memoized dual-search + // is tuned for exactly this case (see perf.md's Shipped try_code finding). + content: Array.from( + {length: 400}, + (_, i) => + `The \`parse_${i}()\` helper takes an \`MdzNode\` and returns \`Array\`; ` + + `see \`mdz_lexer.ts\` and \`try_code\` for the \`buffer_index_of\` memo.`, + ).join('\n'), + }, + { + name: 'nested components', + // The Svelte-component authoring use case (correctly closed + // ``, nested) — the success-path counterpart to the + // adversarial `mismatched tags` / `many angles` inputs, which only + // exercise the never-matching failure path. This drives `try_close_tag`'s + // matching branch and the tag stack under valid nesting. + content: Array.from( + {length: 300}, + (_, i) => + `Section ${i} has **${i}** and a \`ref\` inside.`, + ).join('\n\n'), + }, + { + name: 'nested inline cap', + // The Bug 5 pathology: deeply nested `[` and same-name `
` past the + // inline nesting cap (`MAX_INLINE_NESTING_DEPTH`). Uncapped, the sync + // lexer's rewind-and-retokenize failure path is O(n²) and stack-overflows + // here; the cap bounds it to ≤ cap opens (≤ cap reverts, each re-scanning + // a bounded tail) and the streaming parser caps via `open_link_tag_depth`. + // Tracks the cap path's throughput — the tag half carries a larger + // revert-rescan constant than the bracket half (successful inner tags are + // re-tokenized on each outer revert; see perf.md lead #13). Each unit + // nests ~2× the cap, so every feed strategy exercises both suppression + // and the bounded reverts (cap correctness itself is in + // `mdz_nesting_cap.test.ts`). + content: Array.from( + {length: 12}, + (_, i) => '['.repeat(200) + `text ${i} ` + ''.repeat(200) + `X${i}` + ''.repeat(200), + ).join('\n\n'), + }, +]; diff --git a/src/benchmarks/mdz.benchmark.ts b/src/benchmarks/mdz.benchmark.ts index 166f64f..91df570 100644 --- a/src/benchmarks/mdz.benchmark.ts +++ b/src/benchmarks/mdz.benchmark.ts @@ -19,247 +19,15 @@ import {mdz_parse} from '../lib/mdz.ts'; import {MdzStreamParser} from '../lib/mdz_stream_parser.ts'; import {mdz_opcodes_to_nodes} from '../lib/mdz_opcodes_to_nodes.ts'; import type {MdzOpcode} from '../lib/mdz_opcodes.ts'; +import {benchmark_inputs} from './benchmark_inputs.ts'; const save_baseline = process.argv.includes('--save'); const BASELINE_PATH = 'src/benchmarks'; const BASELINE_FILE = `${BASELINE_PATH}/baseline.json`; -// -- Benchmark inputs -- - -// Generate a large synthetic input -const generate_large_input = (): string => { - const sections: Array = []; - for (let i = 0; i < 50; i++) { - sections.push(`## Section ${i + 1} - -This is paragraph ${i + 1} with **bold** and _italic_ text. -Here's a \`code snippet\` and a [link](https://fuz.dev/${i}). - -\`\`\` -code block ${i + 1} -const value = ${i}; -\`\`\` - -Some more text with https://auto.link/${i} and ~~strikethrough~~ content.`); - } - return `# Large Document\n\n${sections.join('\n\n')}`; -}; - -// Blockquote-dense input: nesting depth changes, in-quote blanks, lists and -// fences inside quotes, quotes inside items, and delimiter-dense quoted -// paragraphs. Guards the nested-parser pipeline — per-line prefix matching, -// one-level stripping, and (streaming) opcode forwarding with offset -// remapping must all stay linear. -const generate_blockquote_heavy_input = (): string => { - const sections: Array = []; - for (let i = 0; i < 40; i++) { - sections.push(`## Quote group ${i + 1} - -> **Reviewer ${i}** wrote about snake_case_idents and ~/dev/path_${i}: -> -> The earlier draft of section_${i} said something _different_ here, -> with more_snake_case and x_${i} markers throughout the paragraph. -> -> - first point about \`api_call_${i}\` -> - second point with **bold** content -> - a nested bullet under it -> -> > and a nested quote quoting reply_${i} -> > across two lines -> -> \`\`\`ts -> const value_${i} = compute(${i}); -> \`\`\` - -1. step with a quoted note - > attached to item ${i} - > continuing the quote -2. follow-up step ${i}`); - } - return `# Blockquote Heavy\n\n${sections.join('\n\n')}`; -}; - -// List-heavy input: deep nesting, blank-line containment, in-item fences, and -// delimiter-dense item content (snake_case `_` failed candidates; the `~/dev/` -// path tildes take the single-`~` literal fast path). Locks in two costs: -// per-line classification inside open lists must stay invisible, and the -// inline closer scans' run-boundary probe must stay amortized-linear on -// delimiter-dense items. -const generate_list_heavy_input = (): string => { - const sections: Array = []; - for (let i = 0; i < 40; i++) { - sections.push(`## Step group ${i + 1} - -1. **Setup ${i}** - - Install the dependency_name_${i} package and set the env_var_${i} value - for the ~/dev/repo_${i} checkout before continuing. - - \`\`\`bash - npm install pkg_${i} - \`\`\` - -2. **Run ${i}** - - nested bullet with snake_case_idents and ~/dev/path_${i} refs - - another _nested_ bullet with more_snake_case here - - deeper level with x_${i} and y_${i} markers - - back out to the sibling level - -3. Final step ${i} with trailing prose - continuing on an indented line`); - } - return `# List Heavy\n\n${sections.join('\n\n')}`; -}; - -// Table-heavy input: many pipe tables with inline-dense cells (bold, code, -// links, strikethrough), code-span and `\|` literal pipes, and per-column -// alignment. Locks in two costs: per-row cell splitting + cell inline parsing -// must stay linear across hundreds of cells (the streaming path shares one -// lexer/parser per row via `MdzTableCellParser`), and the streaming header hold -// (a `|` row held until its delimiter line arrives) must not rescan -// quadratically — the 64B feed strategy splits rows mid-line. -const generate_table_heavy_input = (): string => { - const sections: Array = []; - for (let i = 0; i < 40; i++) { - sections.push(`## Table group ${i + 1} - -| Name | \`Type\` | Default | Notes | -| :--- | :----: | ------: | :---- | -| field_${i}_a | \`string\` | \`""\` | a **bold** note with ./path_${i} | -| field_${i}_b | \`number\` | \`0\` | union \`A_${i} | B_${i}\` protected | -| field_${i}_c | \`boolean\` | \`false\` | escaped a \\| b literal | -| field_${i}_d | \`Array\` | \`[]\` | see [docs](/docs/${i}) and ~~old~~ | -| field_${i}_e | \`Map\` | \`null\` | trailing _italic_ ${i} |`); - } - return `# Table Heavy\n\n${sections.join('\n\n')}`; -}; - -const inputs = [ - {name: 'tiny', content: 'hello **bold** world'}, - { - name: 'small', - content: `# Small Document - -This is a _simple_ paragraph with **bold** text and \`inline code\`. - -Here's a link: [click here](https://fuz.dev) and an auto-link https://fuz.dev/path. - -Some ~~strikethrough~~ text and more _italic_ words.`, - }, - { - name: 'medium', - content: `# Medium Document - -## Introduction - -This is a **medium-sized** document that tests various mdz features. -It has multiple paragraphs with _italic_, **bold**, and \`code\` formatting. - -## Code Examples - -Here's some inline \`code\` and a code block: - -\`\`\`typescript -const x = 42; -function hello() { - return 'world'; -} -\`\`\` - -## Links and References - -Visit [the docs](https://docs.fuz.dev) for more info. -Also see https://fuz.dev/api and /internal/path for details. - -## Formatting - -This paragraph has **bold with _nested italic_** and standalone ~~strikethrough~~ text. -Multiple **bold** words in a **single** line with \`code\` mixed in. - ---- - -Final section after a horizontal rule. - -More content with [multiple](https://a.com) links [here](https://b.com) and [there](/c).`, - }, - {name: 'large', content: generate_large_input()}, - { - name: 'angle brackets', - // Angle brackets without closing tags — exercises tag bail-out paths. - // Before the pre-check fix, unclosed tags like caused O(n*k) - // scanning through the rest of the document. - content: `# TypeScript-Heavy Document - -### Why object literals beat Pick - -A \`Pick\` pattern forces every consumer to import the -god type. Small standalone interfaces have no such coupling. - -## Generic Signatures - -\`\`\`typescript -export interface GitDeps { - checkout: (options: {branch: string}) => Promise>; - push: (options: {cwd?: string}) => Promise>; -} - -export const update = async ( - repos: Array, - updates: Map, -): Promise => {}; - -export const read_json = (path: string): Promise => {}; -\`\`\` - -### Narrowing with \`Pick<>\` - -\`Pick<>\` on small \`*Deps\` interfaces is fine: - -\`\`\`typescript -password: Pick; -\`\`\` - -The anti-pattern is \`Pick\` — coupling every consumer to a large type. - -## More Generics - -Functions with Array, Promise, and Map in prose. - -| Type | Example | -| --- | --- | -| \`Array\` | list of names | -| \`Promise>\` | async result | -| \`Pick\` | narrowed deps |`, - }, - { - name: 'many angles', - // Many unclosed angle brackets in a single paragraph — worst case for - // repeated tag bail-outs within one parse unit - content: Array.from( - {length: 50}, - (_, i) => `Item ${i}: Array and Map> end.`, - ).join('\n'), - }, - {name: 'list heavy', content: generate_list_heavy_input()}, - {name: 'blockquote heavy', content: generate_blockquote_heavy_input()}, - {name: 'table heavy', content: generate_table_heavy_input()}, - { - name: 'large dense inline', - // Large doc dense in unclosed delimiter candidates — ``-shaped - // generics searching for closing tags that never exist. Each failed - // search must not rescan to the end of the document (the `#index_of` - // memo), or parsing goes quadratic and large real-world docs hang. - // The `~/dev` home paths used to be the other half of the pathology - // (single-`~` strikethrough scans); since strikethrough moved to GFM - // `~~`, a single `~` is literal with no scan at all, and the paths now - // exercise that fast path. - content: Array.from( - {length: 1600}, - (_, i) => - `The ~/dev/repo_${i} crate returns Vec and Box while ~/dev/other_${i} uses Option>> in its API surface. `, - ).join('\n'), - }, -]; +// inputs live in `benchmark_inputs.ts`, shared with the render suite +// (`render.benchmark.test.ts`) +const inputs = benchmark_inputs; /** Parse via streaming parser (one-shot feed) + tree bridge. */ const mdz_parse_stream = (content: string): unknown => { @@ -308,13 +76,16 @@ const parsers: Array = [ {name: 'streaming', parse: mdz_parse_stream}, {name: 'opcodes-only', parse: mdz_parse_opcodes_only}, {name: 'streaming 64B', parse: (content) => mdz_parse_stream_chunked(content, 64)}, - // char-by-char is the hardening-#14 worst case (O(n²) buffer string copies - // in feed) — restrict to small inputs so the benchmark stays fast until a - // buffer abstraction lands + // char-by-char is linear on line-bounded input (measured ~4 MB/s — per-feed + // overhead dominates) but still slow in absolute terms; the cap keeps the + // suite's wall clock reasonable while covering every input up to `large` + // and the adversarial hold lines. The residual super-linear term is V8 + // rope flattening while a hold pins the buffer (no compaction), which only + // a chunked buffer abstraction would remove. { name: 'char-by-char', parse: (content) => mdz_parse_stream_chunked(content, 1), - max_input_length: 1000, + max_input_length: 30_000, }, ]; @@ -379,7 +150,13 @@ console.log(bench.summary()); const comparison = await benchmark_baseline_compare(bench.results(), { path: BASELINE_PATH, - regression_threshold: 1.1, // 10% threshold — system-level variance (thermal, scheduler) easily causes 5-8% swings + // Informational only (nothing gates CI on it). Cross-run absolute compare + // can't resolve sub-~2× effects on this hardware — a zero-change re-run has + // flagged most tasks as "regressions" (thermal/scheduler drift, invisible to + // the within-run `noise_warning`). The real signal is within-run scaling / + // same-run ratios (see perf.md "Measurement robustness"); this threshold + // just filters the noisiest local rows. + regression_threshold: 1.1, staleness_warning_days: 30, }); diff --git a/src/benchmarks/render.benchmark.test.ts b/src/benchmarks/render.benchmark.test.ts new file mode 100644 index 0000000..e014c44 --- /dev/null +++ b/src/benchmarks/render.benchmark.test.ts @@ -0,0 +1,132 @@ +/** + * Rendering benchmarks with baseline comparison — SSR through the runtime + * renderers plus the reactive stream consumer (`MdzStreamState.apply_batch`). + * + * Lives in the vitest pipeline (unlike `mdz.benchmark.ts`, which runs under + * `gro run`) because Svelte components and `$state` runes need the Svelte + * compiler; `render()` from `svelte/server` needs no DOM. Gated behind + * `BENCHMARK_RENDER` so ordinary `gro test` runs skip it instantly. + * + * Usage: + * npm run benchmark:render # run and compare against baseline + * npm run benchmark:render:save # run and save as new baseline + * + * The baseline lives at `src/benchmarks/render/baseline.json` — gitignored, + * local-only, like the parser suite's. + */ + +import {describe, test} from 'vitest'; +import {render} from 'svelte/server'; +import type {Component} from 'svelte'; +import {Benchmark} from '@fuzdev/fuz_util/benchmark.ts'; +import { + benchmark_baseline_save, + benchmark_baseline_compare, + benchmark_baseline_format, +} from '@fuzdev/fuz_util/benchmark_baseline.ts'; + +import Mdz from '$lib/Mdz.svelte'; +import MdzStream from '$lib/MdzStream.svelte'; +import {MdzStreamParser} from '$lib/mdz_stream_parser.ts'; +import {MdzStreamState} from '$lib/mdz_stream_state.svelte.ts'; +import type {MdzOpcode} from '$lib/mdz_opcodes.ts'; +import {benchmark_inputs} from './benchmark_inputs.ts'; + +// vitest's reporter intercepts `console.log`; write to stdout directly so the +// tables actually print under `gro test` +const print = (line: string): void => { + process.stdout.write(`${line}\n`); +}; + +const BASELINE_PATH = 'src/benchmarks/render'; + +const mode = process.env.BENCHMARK_RENDER; // undefined | '1' | 'save' + +// the render-relevant subset: block variety (medium/large), the table +// renderer's cell normalization (table heavy), and list/blockquote nesting +const INPUT_NAMES = ['medium', 'large', 'table heavy', 'list heavy', 'blockquote heavy']; +const inputs = benchmark_inputs.filter((i) => INPUT_NAMES.includes(i.name)); +// `name` is a plain string, so a rename/typo in either list silently shrinks +// (or empties) `inputs` with no error — assert the filter matched every name +if (inputs.length !== INPUT_NAMES.length) { + throw new Error( + `render benchmark: INPUT_NAMES matched ${inputs.length}/${INPUT_NAMES.length} inputs — a name drifted`, + ); +} + +const parse_opcodes = (content: string): Array => { + const parser = new MdzStreamParser(); + parser.feed(content); + parser.finish(); + return parser.take_opcodes(); +}; + +// narrowed component types — the wrappers' `SvelteHTMLElements` rest-prop +// union is too complex for `render()`'s generic inference +const MdzNarrow = Mdz as unknown as Component<{content: string}>; +const MdzStreamNarrow = MdzStream as unknown as Component<{stream: MdzStreamState}>; + +describe.skipIf(!mode)('render benchmarks', () => { + test('run', {timeout: 600_000}, async () => { + const bench = new Benchmark({ + duration_ms: 1500, + warmup_iterations: 10, + min_iterations: 30, + }); + + for (const input of inputs) { + // static SSR: parse + render through `Mdz` → `MdzNodeView` + bench.add( + `ssr: ${input.name}`, + () => render(MdzNarrow, {props: {content: input.content}}).body, + ); + + // reactive consumer: opcodes → `MdzStreamState` tree (opcodes + // precomputed — this isolates `apply_batch` from parsing) + const opcodes = parse_opcodes(input.content); + bench.add(`apply: ${input.name}`, () => { + const state = new MdzStreamState(); + state.apply_batch(opcodes); + return state.root.length; + }); + + // streaming SSR: the full pipeline a streaming page pays per render + bench.add(`ssr stream: ${input.name}`, () => { + const state = new MdzStreamState(); + state.apply_batch(parse_opcodes(input.content)); + return render(MdzStreamNarrow, {props: {stream: state}}).body; + }); + } + + await bench.run(); + + print('\n mdz Render Benchmark Results\n'); + print(bench.table()); + print('\n Summary\n'); + print(bench.summary()); + + if (mode === 'save') { + await benchmark_baseline_save(bench.results(), {path: BASELINE_PATH}); + print(`\n✓ Baseline saved to ${BASELINE_PATH}/baseline.json`); + return; + } + + const comparison = await benchmark_baseline_compare(bench.results(), { + path: BASELINE_PATH, + // Informational only (nothing gates CI on it). Cross-run absolute + // compare can't resolve sub-~2× effects on this hardware — a zero-change + // re-run has flagged most tasks as "regressions". The real signal is + // within-run scaling / same-run ratios (see perf.md "Measurement + // robustness"); this threshold just filters the noisiest local rows. + regression_threshold: 1.1, + staleness_warning_days: 30, + }); + print('\n Baseline Comparison\n'); + print(benchmark_baseline_format(comparison)); + if (comparison.baseline_found && comparison.regressions.length > 0) { + print( + '\n⚠️ Regressions detected. Run `npm run benchmark:render:save` to update the baseline if intentional.', + ); + } + }); +}); diff --git a/src/lib/Mdz.svelte b/src/lib/Mdz.svelte index 6952268..24da97b 100644 --- a/src/lib/Mdz.svelte +++ b/src/lib/Mdz.svelte @@ -1,7 +1,8 @@ - {#each nodes as node (node)} - - {/each} + diff --git a/src/lib/MdzNodeView.svelte b/src/lib/MdzNodeView.svelte index 4031c52..71dcc70 100644 --- a/src/lib/MdzNodeView.svelte +++ b/src/lib/MdzNodeView.svelte @@ -2,12 +2,7 @@ import {resolve} from '$app/paths'; import type {MdzNode, MdzNodeTableRow, MdzTableAlign} from './mdz.ts'; - import { - mdz_resolve_relative_path, - mdz_is_safe_reference, - mdz_is_void_element, - } from './mdz_helpers.ts'; - import MdzNodeView from './MdzNodeView.svelte'; + import {mdz_classify_link, mdz_is_void_element} from './mdz_helpers.ts'; import { mdz_components_context, mdz_elements_context, @@ -18,9 +13,10 @@ const { node, - }: { - node: MdzNode; - } = $props(); + nodes, + }: // exactly one of `node`/`nodes` — the array form renders a whole tree + // through this single component instance (used by `Mdz`) + {node: MdzNode; nodes?: undefined} | {node?: undefined; nodes: Array} = $props(); const get_components = mdz_components_context.get_maybe(); const get_elements = mdz_elements_context.get_maybe(); @@ -29,122 +25,132 @@ const get_codeblock = mdz_codeblock_context.get_maybe(); -{#if node.type === 'Element'} - {@const element_config = get_elements?.()?.get(node.name)} - {#if element_config !== undefined} - {#if mdz_is_void_element(node.name)} - - + +{#if nodes} + {@render render_children(nodes)} +{:else if node} + {@render render_node(node)} +{/if} + +{#snippet render_node(node: MdzNode)} + {#if node.type === 'Element'} + {@const element_config = get_elements?.()?.get(node.name)} + {#if element_config !== undefined} + {#if mdz_is_void_element(node.name)} + + + {:else} + + {#if node.children.length > 0} + {@render render_children(node.children)} + {/if} + + {/if} {:else} - + {@render render_unregistered_tag(node.name, node.children)} + {/if} + {:else if node.type === 'Component'} + {@const Component = get_components?.()?.get(node.name)} + {#if Component} + {#if node.children.length > 0} {@render render_children(node.children)} {/if} - + + {:else} + {@render render_unregistered_tag(node.name, node.children)} {/if} - {:else} - {@render render_unregistered_tag(node.name, node.children)} - {/if} -{:else if node.type === 'Component'} - {@const Component = get_components?.()?.get(node.name)} - {#if Component} - - {#if node.children.length > 0} - {@render render_children(node.children)} - {/if} - - {:else} - {@render render_unregistered_tag(node.name, node.children)} - {/if} -{:else if node.type === 'Text'} - {node.content} -{:else if node.type === 'Code'} - {@const Code = get_code?.()} - {#if Code} - - {:else} - {node.content} - {/if} -{:else if node.type === 'Bold'} - {@render render_children(node.children)} -{:else if node.type === 'Italic'} - {@render render_children(node.children)} -{:else if node.type === 'Strikethrough'} - {@render render_children(node.children)} -{:else if node.type === 'Link'} - {@const {reference} = node} - {#if !mdz_is_safe_reference(reference)} - {@render render_children(node.children)} - {:else if node.link_type === 'internal'} - {@const skip_resolve = reference.startsWith('#') || reference.startsWith('?')} - {@const mdz_base = get_mdz_base?.()} - {#if reference.startsWith('.') && mdz_base} - {@const resolved = mdz_resolve_relative_path(reference, mdz_base)} - {@render render_children(node.children)} - {:else if skip_resolve || reference.startsWith('.') || !reference.startsWith('/')} - - - - {@render render_children(node.children)} + {:else if node.type === 'Text'} + {node.content} + {:else if node.type === 'Code'} + {@const Code = get_code?.()} + {#if Code} + {:else} - {@render render_children(node.children)} + {node.content} {/if} - {:else} - - - {@render render_children(node.children)} - {/if} -{:else if node.type === 'Paragraph'} -

{@render render_children(node.children)}

-{:else if node.type === 'List'} - {#if node.ordered} -
    + {:else if node.type === 'Bold'} + {@render render_children(node.children)} + {:else if node.type === 'Italic'} + {@render render_children(node.children)} + {:else if node.type === 'Strikethrough'} + {@render render_children(node.children)} + {:else if node.type === 'Link'} + {@const link = mdz_classify_link(node.reference, node.link_type, get_mdz_base?.())} + {#if link.kind === 'unsafe'} {@render render_children(node.children)} -
- {:else} -
    {@render render_children(node.children)}
- {/if} -{:else if node.type === 'ListItem'} -
  • {@render render_children(node.children)}
  • -{:else if node.type === 'Blockquote'} -
    {@render render_children(node.children)}
    -{:else if node.type === 'Table'} - {@const header_rows = node.children.filter((r) => r.header)} - {@const body_rows = node.children.filter((r) => !r.header)} - - {#if header_rows.length > 0} - - {#each header_rows as row (row)} - {@render render_cells(row, 'th', node.align)} - {/each} - + {:else if link.kind === 'resolve'} + {@render render_children(node.children)} + {:else if link.kind === 'external'} + + {@render render_children(node.children)} + {:else} + + + {@render render_children(node.children)} {/if} - {#if body_rows.length > 0} - - {#each body_rows as row (row)} - {@render render_cells(row, 'td', node.align)} - {/each} - + {:else if node.type === 'Paragraph'} +

    {@render render_children(node.children)}

    + {:else if node.type === 'List'} + {#if node.ordered} +
      + {@render render_children(node.children)} +
    + {:else} +
      {@render render_children(node.children)}
    + {/if} + {:else if node.type === 'ListItem'} +
  • {@render render_children(node.children)}
  • + {:else if node.type === 'Blockquote'} +
    {@render render_children(node.children)}
    + {:else if node.type === 'Table'} + + {@const header_row = node.children[0]?.header ? node.children[0] : undefined} +
    + {#if header_row} + + {@render render_cells(header_row, 'th', node.align)} + + {/if} + {#if node.children.length > (header_row ? 1 : 0)} + + {#each node.children as row, i (i)} + {#if !row.header} + {@render render_cells(row, 'td', node.align)} + {/if} + {/each} + + {/if} +
    + {:else if node.type === 'Hr'} +
    + {:else if node.type === 'Heading'} + + {@render render_children(node.children)} + + {:else if node.type === 'Codeblock'} + {@const Codeblock = get_codeblock?.()} + {#if Codeblock} + + {:else} +
    {node.content}
    {/if} - -{:else if node.type === 'Hr'} -
    -{:else if node.type === 'Heading'} - - {@render render_children(node.children)} - -{:else if node.type === 'Codeblock'} - {@const Codeblock = get_codeblock?.()} - {#if Codeblock} - - {:else} -
    {node.content}
    {/if} -{/if} +{/snippet} {#snippet render_children(nodes: Array)} - {#each nodes as node (node)} - + + {#each nodes as node, i (i)} + {@render render_node(node)} {/each} {/snippet} diff --git a/src/lib/MdzStream.svelte b/src/lib/MdzStream.svelte index e49c8a1..86de832 100644 --- a/src/lib/MdzStream.svelte +++ b/src/lib/MdzStream.svelte @@ -31,7 +31,7 @@ - {#each stream.root as node (node.id)} - - {/each} + + diff --git a/src/lib/MdzStreamNodeView.svelte b/src/lib/MdzStreamNodeView.svelte index 337232b..acc32f0 100644 --- a/src/lib/MdzStreamNodeView.svelte +++ b/src/lib/MdzStreamNodeView.svelte @@ -1,12 +1,7 @@ -{#if node.type === 'Element'} - {@const element_config = get_elements?.()?.get(node.name ?? '')} - {#if element_config !== undefined} - {#if mdz_is_void_element(node.name ?? '')} - - + +{#if nodes} + {@render render_children(nodes)} +{:else if node} + {@render render_node(node)} +{/if} + +{#snippet render_node(node: MdzStreamNode)} + {#if node.type === 'Element'} + {@const element_config = get_elements?.()?.get(node.name ?? '')} + {#if element_config !== undefined} + {#if mdz_is_void_element(node.name ?? '')} + + + {:else} + + {#if node.children.length > 0} + {@render render_children(node.children)} + {/if} + + {/if} {:else} - + {@render render_unregistered_tag(node.name ?? '', node.children)} + {/if} + {:else if node.type === 'Component'} + {@const Component = get_components?.()?.get(node.name ?? '')} + {#if Component} + {#if node.children.length > 0} {@render render_children(node.children)} {/if} - + + {:else} + {@render render_unregistered_tag(node.name ?? '', node.children)} {/if} - {:else} - {@render render_unregistered_tag(node.name ?? '', node.children)} - {/if} -{:else if node.type === 'Component'} - {@const Component = get_components?.()?.get(node.name ?? '')} - {#if Component} - - {#if node.children.length > 0} - {@render render_children(node.children)} - {/if} - - {:else} - {@render render_unregistered_tag(node.name ?? '', node.children)} - {/if} -{:else if node.type === 'Text'} - {node.content} -{:else if node.type === 'Code'} - {@const Code = get_code?.()} - {@const code_content = streaming_content(node)} - {#if Code} - - {:else} - {code_content} - {/if} -{:else if node.type === 'Bold'} - {@render render_children(node.children)} -{:else if node.type === 'Italic'} - {@render render_children(node.children)} -{:else if node.type === 'Strikethrough'} - {@render render_children(node.children)} -{:else if node.type === 'Link'} - {@const reference = node.reference ?? ''} - {#if !mdz_is_safe_reference(reference)} - {@render render_children(node.children)} - {:else if node.link_type === 'internal'} - {@const skip_resolve = reference.startsWith('#') || reference.startsWith('?')} - {@const mdz_base = get_mdz_base?.()} - {#if reference.startsWith('.') && mdz_base} - {@const resolved = mdz_resolve_relative_path(reference, mdz_base)} - {@render render_children(node.children)} - {:else if skip_resolve || reference.startsWith('.') || !reference.startsWith('/')} - - - - {@render render_children(node.children)} + {:else if node.type === 'Text'} + {node.content} + {:else if node.type === 'Code'} + {@const Code = get_code?.()} + {@const code_content = streaming_content(node)} + {#if Code} + {:else} - {@render render_children(node.children)} + {code_content} {/if} - {:else} - - - {@render render_children(node.children)} - {/if} -{:else if node.type === 'Paragraph'} -

    {@render render_children(node.children)}

    -{:else if node.type === 'List'} - {#if node.ordered} -
      + {:else if node.type === 'Bold'} + {@render render_children(node.children)} + {:else if node.type === 'Italic'} + {@render render_children(node.children)} + {:else if node.type === 'Strikethrough'} + {@render render_children(node.children)} + {:else if node.type === 'Link'} + {@const link = mdz_classify_link(node.reference ?? '', node.link_type, get_mdz_base?.())} + {#if link.kind === 'unsafe'} {@render render_children(node.children)} -
    - {:else} -
      {@render render_children(node.children)}
    - {/if} -{:else if node.type === 'ListItem'} -
  • {@render render_children(node.children)}
  • -{:else if node.type === 'Blockquote'} -
    {@render render_children(node.children)}
    -{:else if node.type === 'Table'} - {@const align = node.align ?? []} - {@const header_rows = node.children.filter((r) => r.header)} - {@const body_rows = node.children.filter((r) => !r.header)} - - {#if header_rows.length > 0} - - {#each header_rows as row (row.id)} - {@render render_cells(row, 'th', align)} - {/each} - + {:else if link.kind === 'resolve'} + {@render render_children(node.children)} + {:else if link.kind === 'external'} + + {@render render_children(node.children)} + {:else} + + + {@render render_children(node.children)} {/if} - {#if body_rows.length > 0} - - {#each body_rows as row (row.id)} - {@render render_cells(row, 'td', align)} - {/each} - + {:else if node.type === 'Paragraph'} +

    {@render render_children(node.children)}

    + {:else if node.type === 'List'} + {#if node.ordered} +
      + {@render render_children(node.children)} +
    + {:else} +
      {@render render_children(node.children)}
    + {/if} + {:else if node.type === 'ListItem'} +
  • {@render render_children(node.children)}
  • + {:else if node.type === 'Blockquote'} +
    {@render render_children(node.children)}
    + {:else if node.type === 'Table'} + {@const align = node.align ?? []} + + {@const header_row = node.children[0]?.header ? node.children[0] : undefined} +
    + {#if header_row} + + {@render render_cells(header_row, 'th', align)} + + {/if} + {#if node.children.length > (header_row ? 1 : 0)} + + {#each node.children as row (row.id)} + {#if !row.header} + {@render render_cells(row, 'td', align)} + {/if} + {/each} + + {/if} +
    + {:else if node.type === 'Hr'} +
    + {:else if node.type === 'Heading'} + + {@render render_children(node.children)} + + {:else if node.type === 'Codeblock'} + {@const Codeblock = get_codeblock?.()} + {@const codeblock_content = streaming_content(node)} + {#if Codeblock} + + {:else} +
    {codeblock_content}
    {/if} - -{:else if node.type === 'Hr'} -
    -{:else if node.type === 'Heading'} - - {@render render_children(node.children)} - -{:else if node.type === 'Codeblock'} - {@const Codeblock = get_codeblock?.()} - {@const codeblock_content = streaming_content(node)} - {#if Codeblock} - - {:else} -
    {codeblock_content}
    {/if} -{/if} +{/snippet} {#snippet render_children(nodes: Array)} {#each nodes as node (node.id)} - + {@render render_node(node)} {/each} {/snippet} diff --git a/src/lib/mdz.ts b/src/lib/mdz.ts index 22f9135..8bf527f 100644 --- a/src/lib/mdz.ts +++ b/src/lib/mdz.ts @@ -89,6 +89,15 @@ export class MdzTableCellParser { this.#lexer = new MdzLexer(text); } + /** + * Rebind to a new buffer so one instance serves a whole table — the + * streaming path rebinds per row (the buffer string changes across feeds; + * the lexer's search memo survives when it doesn't). + */ + rebind(text: string): void { + this.#lexer.rebind(text); + } + parse(cell_start: number, cell_end: number): Array { this.#parser.reset(this.#lexer.lex_table_cell(cell_start, cell_end)); return this.#parser.parse_inline_run(); diff --git a/src/lib/mdz_debug_work.ts b/src/lib/mdz_debug_work.ts new file mode 100644 index 0000000..9cc3c0b --- /dev/null +++ b/src/lib/mdz_debug_work.ts @@ -0,0 +1,38 @@ +/** + * Deterministic scan-work counter for the complexity-ratio guards + * (`mdz_scaling_ratio.test.ts`). Off by default; when `enabled`, the parser hot + * paths accumulate the number of characters scanned (and tag-stack frames + * walked) into `total`. Unlike wall-clock timing this is a pure function of + * input + algorithm — invariant across machines, runs, and thermal state — so a + * super-linear regression shows up as a work ratio > ~linear regardless of the + * ~2× machine variance that makes absolute timing unusable here. Its sole + * consumer is `mdz_scaling_ratio.test.ts`. + * + * Coverage is the dominant re-scan choke points — `#index_of` + `#tokenize_text` + * (sync) and the streaming memo scans, `consume_text_run`, and `try_close_tag`'s + * stack walk — not every scan. A super-linear regression isolated to an + * *uncounted* probe (e.g. `#has_paragraph_break_between`, the tag-name collection + * loop, the URL/path probes) wouldn't move the ratio; extend the instrumentation + * if hot cost ever migrates there. + * + * Zero store when disabled: each instrumented site guards on `enabled` (a single + * branch-predicted property load, the same category as the shipped `DEV` + * checks). It is deliberately NOT the `esm-env` `DEV` flag — `DEV` is true under + * every `gro test`/`gro dev` run, which would leave the counter live in ordinary + * development; this flag is flipped true only by the ratio test. + * + * @nodocs + */ +export const mdz_debug_work: {enabled: boolean; total: number} = {enabled: false, total: 0}; + +/** + * Reset the counter and set whether the hot paths accumulate into it. Tests call + * `mdz_debug_work_reset()` immediately before a parse and read + * `mdz_debug_work.total` immediately after; pass `false` to disable and zero it. + * + * @nodocs + */ +export const mdz_debug_work_reset = (enabled = true): void => { + mdz_debug_work.enabled = enabled; + mdz_debug_work.total = 0; +}; diff --git a/src/lib/mdz_helpers.ts b/src/lib/mdz_helpers.ts index 925a1ff..626ad6c 100644 --- a/src/lib/mdz_helpers.ts +++ b/src/lib/mdz_helpers.ts @@ -99,6 +99,23 @@ export const MIN_CODEBLOCK_BACKTICKS = 3; // Code blocks require minimum 3 backt export const MAX_LIST_NUMBER_DIGITS = 9; // Ordered list markers cap at 9 digits (CommonMark's cap) /** @nodocs */ export const MAX_HEADING_LEVEL = 6; // Headings support levels 1-6 +/** + * Cap on inline link/tag nesting depth. Reached at this many open + * `Link`/`Element`/`Component` containers, a further `[` or `` opener + * renders as literal text in both parsers rather than opening — a strict + * untrusted-content dialect's bound on pathological nesting (CommonMark + * permits such a limit). It bounds both the sync lexer's recursion depth (no + * stack overflow) and, with the failure memo, its revert re-scan work (linear + * instead of quadratic) on adversarial nested `[`/`<`. Deliberately far above + * any real content (inline nesting rarely exceeds a handful); only pathological + * input reaches it. Mirrored in the streaming parser so the two agree on where + * the cap bites. Delimiters (`**`/`_`/`~~`/`` ` ``) are uncounted: first-closer-wins + * keeps them from nesting deeply on their own, and mixed delimiter/link chains + * only deep-nest through the links, which this bounds. + * + * @nodocs + */ +export const MAX_INLINE_NESTING_DEPTH = 100; /** @nodocs */ export const HTTPS_PREFIX_LENGTH = 8; // Length of "https://" /** @nodocs */ @@ -547,6 +564,40 @@ const PATH_CHAR_TABLE: Uint8Array = (() => { export const is_valid_path_char = (char_code: number): boolean => char_code < 128 && PATH_CHAR_TABLE[char_code] === 1; +// Character classes for the plain-text run scanners (the hottest loops in +// both parsers) — one table load per character instead of a comparison chain. +// A zero class is the common case: the char can't end or interrupt a text run. + +/** Always ends a text run: `` ` `` `*` `_` `~` `[` `]` `<`. @nodocs */ +export const TEXT_CLASS_STOP = 1; +/** `\n` — run handling is context-dependent (soft break, paragraph break, block lookahead). @nodocs */ +export const TEXT_CLASS_NEWLINE = 2; +/** `h`/`H` — potential URL scheme start, needs the prefix probe. @nodocs */ +export const TEXT_CLASS_URL = 4; +/** `/` and `.` — potential path start, needs the boundary probe. @nodocs */ +export const TEXT_CLASS_PATH = 8; +/** `\` — a literal-pipe escape candidate inside table cells only. @nodocs */ +export const TEXT_CLASS_BACKSLASH = 16; + +/** @nodocs */ +export const TEXT_SCAN_CLASS: Uint8Array = (() => { + const t = new Uint8Array(128); + t[BACKTICK] = TEXT_CLASS_STOP; + t[ASTERISK] = TEXT_CLASS_STOP; + t[UNDERSCORE] = TEXT_CLASS_STOP; + t[TILDE] = TEXT_CLASS_STOP; + t[LEFT_BRACKET] = TEXT_CLASS_STOP; + t[RIGHT_BRACKET] = TEXT_CLASS_STOP; + t[LEFT_ANGLE] = TEXT_CLASS_STOP; + t[NEWLINE] = TEXT_CLASS_NEWLINE; + t[H_LOWER] = TEXT_CLASS_URL; + t[H_UPPER] = TEXT_CLASS_URL; + t[SLASH] = TEXT_CLASS_PATH; + t[PERIOD] = TEXT_CLASS_PATH; + t[BACKSLASH] = TEXT_CLASS_BACKSLASH; + return t; +})(); + /** * Trim trailing punctuation from URL/path per RFC 3986 and GFM rules. * - Trims simple trailing: .,;:!?] @@ -760,6 +811,52 @@ export const mdz_resolve_relative_path = (reference: string, base: string): stri return segments.join('/'); }; +/** + * How a `Link` node's `reference` should render, shared by all three renderers + * (`MdzNodeView`, `MdzStreamNodeView`, `mdz_to_svelte`) so the safety gate and + * the resolve-vs-raw classification can't drift between them. + * @nodocs + */ +export type MdzLinkRender = + /** Unsafe protocol — render children only, no ``. */ + | {kind: 'unsafe'} + /** Route/relative path — wrap in `resolve()` (needs `$app/paths`). */ + | {kind: 'resolve'; href: string} + /** Internal fragment/query/relative/bare ref — raw `href`, no `resolve()`. */ + | {kind: 'internal'; href: string} + /** External link — raw `href` plus `target="_blank" rel="noopener"`. */ + | {kind: 'external'; href: string}; + +/** + * Classify a `Link` reference into how it should render. Pure — no rendering, + * no context; `base` is the resolved base path (or `undefined`). + * @nodocs + */ +export const mdz_classify_link = ( + reference: string, + link_type: 'internal' | 'external' | undefined, + base: string | undefined, +): MdzLinkRender => { + if (!mdz_is_safe_reference(reference)) return {kind: 'unsafe'}; + if (link_type === 'internal') { + if (reference.startsWith('.') && base) { + return {kind: 'resolve', href: mdz_resolve_relative_path(reference, base)}; + } + // fragment/query/relative/bare — `resolve()` only accepts absolute paths or + // route ids and throws on anything else + if ( + reference.startsWith('#') || + reference.startsWith('?') || + reference.startsWith('.') || + !reference.startsWith('/') + ) { + return {kind: 'internal', href: reference}; + } + return {kind: 'resolve', href: reference}; + } + return {kind: 'external', href: reference}; +}; + /** * Push a node into a children array, coalescing with the previous Text node. * Mutates `prev.content` and `prev.end` when both are Text, avoiding array growth diff --git a/src/lib/mdz_lexer.ts b/src/lib/mdz_lexer.ts index 3be2b7b..91fba4c 100644 --- a/src/lib/mdz_lexer.ts +++ b/src/lib/mdz_lexer.ts @@ -38,6 +38,7 @@ import { HR_HYPHEN_COUNT, MIN_CODEBLOCK_BACKTICKS, MAX_HEADING_LEVEL, + MAX_INLINE_NESTING_DEPTH, MAX_LIST_NUMBER_DIGITS, match_url_prefix_case_insensitive, is_at_absolute_path, @@ -50,10 +51,15 @@ import { mdz_split_table_row, mdz_parse_table_delimiter, PIPE, - BACKSLASH, + TEXT_CLASS_BACKSLASH, + TEXT_CLASS_NEWLINE, + TEXT_CLASS_STOP, + TEXT_CLASS_URL, + TEXT_SCAN_CLASS, } from './mdz_helpers.ts'; import type {MdzTableAlign} from './mdz.ts'; +import {mdz_debug_work} from './mdz_debug_work.ts'; // // Token types @@ -304,11 +310,69 @@ export class MdzLexer { * and `\|` unescapes to a literal pipe. */ #in_table_cell: boolean = false; + /** + * Start offsets of markdown-link opens (`[`) that definitively failed — so + * a re-lex from before the `[` short-circuits to literal text instead of + * re-attempting (and re-descending into) the same failing link. Without it, + * the revert-and-re-tokenize failure path is exponential on nested `[` + * (a ~30-byte `[`×n input hangs). A link's outcome (does a valid `](ref)` + * follow?) is independent of `#max_search_index`, so `start` alone keys it. + * The outcome is a pure function of the immutable text, so the memo is valid + * for the whole lex and across table-cell ranges; cleared on `rebind`. + */ + #failed_link_starts: Set = new Set(); + /** + * The tag analogue of `#failed_link_starts`, keyed `${start}:${bound}`. A + * tag's closer search respects `#max_search_index`, so the same `start` can + * fail under a narrow bound yet succeed under the wider one a re-lex uses — + * the bound is part of the key. + */ + #failed_tag: Set = new Set(); + /** + * Current link/tag nesting depth — the number of open + * `link_text_open`/`tag_open` containers being recursed into. Bumped around + * `#tokenize_markdown_link`/`#tokenize_tag`'s children loops (not by + * delimiters, which can't nest deeply — see `MAX_INLINE_NESTING_DEPTH`). At + * the cap a further `[`/`<` renders literal instead of opening, bounding + * recursion depth (no stack overflow) and, with the failure memos, the + * revert re-scan (linear, not quadratic) on adversarial nesting. + */ + #inline_depth: number = 0; + /** + * Start offsets of `[`/`<` openers suppressed by the nesting-depth cap + * (`#inline_depth >= MAX_INLINE_NESTING_DEPTH`) — position-keyed like + * `#failed_link_starts` so a re-lex from a shallower depth short-circuits to + * literal text rather than re-opening the construct (which would reintroduce + * the quadratic the cap removes). A suppression is bound-independent, so + * `start` alone keys it; cleared on `rebind`. + */ + #depth_capped: Set = new Set(); constructor(text: string) { this.#text = text; } + /** + * Rebind to a new text, so one lexer instance can be reused across inputs — + * the streaming table path reuses one across a table's rows via + * `MdzTableCellParser` instead of allocating a lexer (and its memo `Map`) + * per row. The search memos survive when the text is identical (they're + * valid over the immutable text — the common case for rows parsed from one + * buffered chunk) and clear otherwise. Per-parse cursor state is reset by + * `lex_table_cell` itself. + * + * @nodocs + */ + rebind(text: string): void { + if (this.#text === text) return; + this.#text = text; + this.#search_memo.clear(); + this.#break_memo = null; + this.#failed_link_starts.clear(); + this.#failed_tag.clear(); + this.#depth_capped.clear(); + } + /** * Memoized `String#indexOf` over the immutable text. * @@ -325,6 +389,10 @@ export class MdzLexer { return memo.result; } const result = this.#text.indexOf(needle, from); + if (mdz_debug_work.enabled) { + // scan span: `indexOf` examined [from, result] (found) or [from, EOF) + mdz_debug_work.total += (result === -1 ? this.#text.length : result) - from; + } if (memo === undefined) { this.#search_memo.set(needle, {from, result}); } else { @@ -376,6 +444,9 @@ export class MdzLexer { */ lex_table_cell(cell_start: number, cell_end: number): Array { this.#tokens = []; + // depth is balanced back to 0 per cell, but pin it so a future early-return + // between an `#inline_depth++` and its decrement can't leak into the next cell + this.#inline_depth = 0; this.#tokenize_cell_inline(cell_start, cell_end); return this.#tokens; } @@ -1541,6 +1612,23 @@ export class MdzLexer { #tokenize_markdown_link(): void { const start = this.#index; + // a link open at this offset already failed, or was suppressed by the + // nesting cap — emit `[` as text instead of re-descending into the same + // failing children (exponential on nested `[`); the short-circuit + // reproduces the revert's exact net effect (`[` text, cursor at start + 1) + if (this.#failed_link_starts.has(start) || this.#depth_capped.has(start)) { + this.#index = start + 1; + this.#emit_text('[', start); + return; + } + // nesting-depth cap: too deep to open — render literal and record the + // suppression so a shallower re-lex short-circuits here too + if (this.#inline_depth >= MAX_INLINE_NESTING_DEPTH) { + this.#depth_capped.add(start); + this.#index = start + 1; + this.#emit_text('[', start); + return; + } this.#index++; // consume [ (dispatch guarantees it) // Emit link_text_open @@ -1549,11 +1637,13 @@ export class MdzLexer { // Tokenize children until ] — only `]` delimits link text; a bare `)` // is ordinary content (matching the streaming parser, which has no `)` // dispatch case — the reference scan finds its own `)` after `](`) + this.#inline_depth++; while (this.#index < this.#text.length) { if (this.#text.charCodeAt(this.#index) === RIGHT_BRACKET) break; if (this.#is_at_paragraph_break()) break; this.#tokenize_inline(); } + this.#inline_depth--; // Check for ] if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_BRACKET) { @@ -1622,6 +1712,13 @@ export class MdzLexer { } #revert_tokens_from_link_open(start: number): void { + // record the failure so a re-lex from before this `[` short-circuits. + // Keyed by `start` alone — unlike `#failed_tag`'s `start:bound` — because a + // link's outcome is bound-independent: the `]`-terminator scan (the children + // loop) runs to `text.length` and never consults `#max_search_index`, so a + // wider-bound re-lex can't move it. The bound reaches nested tags inside the + // link text, but re-attempting those wider never flips the link's net result. + this.#failed_link_starts.add(start); // Find and remove link_text_open and all tokens after it let open_idx = -1; for (let i = this.#tokens.length - 1; i >= 0; i--) { @@ -1639,6 +1736,32 @@ export class MdzLexer { #tokenize_tag(): void { const start = this.#index; + // short-circuit a re-attempt of a tag open that already failed at this + // offset+bound — re-descending into the same failing children is + // exponential on nested same-name tags. Keyed by + // `#max_search_index` (captured as `bound`) because the closer search is + // bound-sensitive, so a wider-bound re-lex can legitimately succeed. The + // key string is built only when the memo is non-empty (adversarial + // input); prose never allocates it. The short-circuit reproduces a + // revert's exact net effect (`<` text, cursor at start + 1). + const bound = this.#max_search_index; + if ( + this.#depth_capped.has(start) || + (this.#failed_tag.size !== 0 && this.#failed_tag.has(`${start}:${bound}`)) + ) { + this.#index = start + 1; + this.#emit_text('<', start); + return; + } + // nesting-depth cap: too deep to open a tag — render literal and record + // the suppression (position-keyed) so a shallower re-lex short-circuits + // here too, keeping the revert re-scan linear + if (this.#inline_depth >= MAX_INLINE_NESTING_DEPTH) { + this.#depth_capped.add(start); + this.#index = start + 1; + this.#emit_text('<', start); + return; + } this.#index++; // consume < // Tag name must start with a letter @@ -1685,6 +1808,7 @@ export class MdzLexer { // Check for > if (this.#index >= this.#text.length || this.#text.charCodeAt(this.#index) !== RIGHT_ANGLE) { + this.#failed_tag.add(`${start}:${bound}`); this.#index = start + 1; this.#emit_text('<', start); return; @@ -1703,11 +1827,13 @@ export class MdzLexer { const search_limit = Math.min(this.#max_search_index, this.#text.length); const closing_tag_pos = this.#index_of(closing_tag, this.#index); if (closing_tag_pos === -1 || closing_tag_pos > search_limit) { + this.#failed_tag.add(`${start}:${bound}`); this.#index = start + 1; this.#emit_text('<', start); return; } if (this.#has_paragraph_break_between(this.#index, closing_tag_pos)) { + this.#failed_tag.add(`${start}:${bound}`); this.#index = start + 1; this.#emit_text('<', start); return; @@ -1723,8 +1849,10 @@ export class MdzLexer { // the code span). Recomputed per iteration because a nested same-name // tag legitimately consumes the nearest closer (`xy`). const saved_max = this.#max_search_index; + this.#inline_depth++; while (this.#index < this.#text.length) { if (this.#match(closing_tag)) { + this.#inline_depth--; this.#max_search_index = saved_max; const close_start = this.#index; this.#index += closing_tag.length; @@ -1747,12 +1875,14 @@ export class MdzLexer { this.#max_search_index = next_close; this.#tokenize_inline(); } + this.#inline_depth--; // No valid closer left — revert: drop the open token and all consumed // children tokens, emit `<` as text, and re-lex from the next char // (the `#revert_tokens_from_link_open` pattern; the old fallthrough // left a dangling `tag_open` whose parser fallback discarded content) this.#max_search_index = saved_max; + this.#failed_tag.add(`${start}:${bound}`); this.#tokens.splice(open_token_index); this.#index = start + 1; this.#emit_text('<', start); @@ -1780,16 +1910,33 @@ export class MdzLexer { let parts: Array | null = null; let part_start = start; - while (this.#index < this.#text.length) { + // in a table cell the trimmed cell end is a hard stop (no newline to + // halt on mid-line) — hoisted to the loop bound + const scan_end = this.#in_table_cell + ? Math.min(this.#max_search_index, this.#text.length) + : this.#text.length; + + while (this.#index < scan_end) { const char_code = this.#text.charCodeAt(this.#index); + // one table load classifies the char; zero (the common case — the + // char can't end or interrupt the run) skips the dispatch below. + // `TEXT_SCAN_CLASS` is shared with the streaming scanner + // (`consume_text_run` in `mdz_stream_parser_text.ts`); the two dispatch + // bodies intentionally differ around it — this one handles `\|` escapes + // and list-run strip, that one URL speculation across chunk boundaries. + const char_class = char_code < 128 ? TEXT_SCAN_CLASS[char_code]! : 0; + if (char_class === 0) { + this.#index++; + continue; + } + + // Stop at special characters + if ((char_class & TEXT_CLASS_STOP) !== 0) break; - // in a table cell: the trimmed cell end is a hard stop (no newline to - // halt on mid-line), and `\|` is a literal pipe (the only mdz escape, - // scoped here) - if (this.#in_table_cell) { - if (this.#index >= this.#max_search_index) break; + if ((char_class & TEXT_CLASS_BACKSLASH) !== 0) { + // `\|` is a literal pipe — the only mdz escape, scoped to table cells if ( - char_code === BACKSLASH && + this.#in_table_cell && this.#index + 1 < this.#max_search_index && this.#text.charCodeAt(this.#index + 1) === PIPE ) { @@ -1799,22 +1946,11 @@ export class MdzLexer { part_start = this.#index; continue; } + this.#index++; + continue; } - // Stop at special characters - if ( - char_code === BACKTICK || - char_code === ASTERISK || - char_code === UNDERSCORE || - char_code === TILDE || - char_code === LEFT_BRACKET || - char_code === RIGHT_BRACKET || - char_code === LEFT_ANGLE - ) { - break; - } - - if (char_code === NEWLINE) { + if ((char_class & TEXT_CLASS_NEWLINE) !== 0) { if (this.#list_run_strip) { if (this.#index >= this.#max_search_index) break; // the run's terminator // soft break: keep the newline, strip the next line's leading @@ -1853,20 +1989,24 @@ export class MdzLexer { break; } } + this.#index++; // single newline — ordinary text + continue; } - // Check for URL or internal path mid-text (char code guard avoids startsWith on every char) - if ( - ((char_code === 104 /* h */ || char_code === 72) /* H */ && this.#is_at_url()) || - (char_code === SLASH && is_at_absolute_path(this.#text, this.#index)) || - (char_code === PERIOD && is_at_relative_path(this.#text, this.#index)) - ) { + // URL or internal path mid-text (class guard avoids probing every char) + if ((char_class & TEXT_CLASS_URL) !== 0) { + if (this.#is_at_url()) break; + } else if (char_code === SLASH) { + if (is_at_absolute_path(this.#text, this.#index)) break; + } else if (is_at_relative_path(this.#text, this.#index)) { break; } this.#index++; } + if (mdz_debug_work.enabled) mdz_debug_work.total += this.#index - start; + // Ensure we always consume at least one character if (this.#index === start && this.#index < this.#text.length) { this.#index++; diff --git a/src/lib/mdz_stream_parser_inline.ts b/src/lib/mdz_stream_parser_inline.ts index 9aa1884..f907721 100644 --- a/src/lib/mdz_stream_parser_inline.ts +++ b/src/lib/mdz_stream_parser_inline.ts @@ -339,17 +339,24 @@ export const scan_closer_boundary = ( /** @nodocs */ export const try_code = (state: MdzStreamParserState, forced = false): boolean => { const start = state.pos; - let i = start + 1; // past opening backtick + const search_start = start + 1; // past opening backtick // compute search boundary: don't scan past open formatting delimiters - const search_limit = code_search_limit(state, start + 1); + const search_limit = code_search_limit(state, search_start); - // scan for closing backtick (must be before newline and search boundary) - while (i < state.buffer.length && i < search_limit) { - const c = state.buffer.charCodeAt(i); - if (c === BACKTICK) { + // find the first closing backtick or newline (memoized — a candidate that + // holds under open formatting re-enters here on every feed, and a raw scan + // would re-cover the same growing region each time, going quadratic on a + // single long line) + const close_pos = buffer_index_of(state, '`', search_start); + const newline_pos = buffer_index_of(state, '\n', search_start); + const terminator = + close_pos !== -1 && (newline_pos === -1 || close_pos < newline_pos) ? close_pos : newline_pos; + + if (terminator !== -1 && terminator < search_limit) { + if (terminator === close_pos) { // found close - const content = state.buffer.slice(start + 1, i); + const content = state.buffer.slice(search_start, terminator); if (content.length === 0) { // empty code — treat as text consume_delimiter_as_text(state, '``'); @@ -364,20 +371,17 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean = content, text_type: 'Code', start: offset(state, start), - end: offset(state, i + 1), + end: offset(state, terminator + 1), }); state.active_text_id = null; - state.pos = i + 1; - state.column += i + 1 - start; + state.pos = terminator + 1; + state.column += terminator + 1 - start; state.prev_char = BACKTICK; return true; } - if (c === NEWLINE) { - // inline code can't span lines — treat opening backtick as text - consume_delimiter_as_text(state, '`'); - return true; - } - i++; + // newline before any close — inline code can't span lines + consume_delimiter_as_text(state, '`'); + return true; } // hit search limit (open formatting delimiter boundary) — treat backtick as @@ -386,7 +390,7 @@ export const try_code = (state: MdzStreamParserState, forced = false): boolean = // processed), the one-shot parse rejects that italic and scans unbounded — // see the `MdzStreamParser` TSDoc and the parity suite's expected-divergence // battery - if (i < state.buffer.length && i >= search_limit) { + if (search_limit < state.buffer.length) { consume_delimiter_as_text(state, '`'); return true; } diff --git a/src/lib/mdz_stream_parser_link.ts b/src/lib/mdz_stream_parser_link.ts index 0ea8dcc..262e7e5 100644 --- a/src/lib/mdz_stream_parser_link.ts +++ b/src/lib/mdz_stream_parser_link.ts @@ -10,7 +10,7 @@ import { A_UPPER, LEFT_BRACKET, LEFT_PAREN, - NEWLINE, + MAX_INLINE_NESTING_DEPTH, RIGHT_ANGLE, RIGHT_BRACKET, RIGHT_PAREN, @@ -28,6 +28,7 @@ import { type TryResult, accumulate_text, alloc_id, + buffer_index_of, emit, ensure_paragraph, flush_text, @@ -37,11 +38,20 @@ import { push_stack_entry, revert_above, } from './mdz_stream_parser_state.ts'; +import {consume_delimiter_as_text} from './mdz_stream_parser_text.ts'; +import {mdz_debug_work} from './mdz_debug_work.ts'; // -- Links -- /** @nodocs */ export const try_link_open = (state: MdzStreamParserState): boolean => { + // nesting-depth cap: too deep to open — render `[` literal, matching the + // sync lexer's `#tokenize_markdown_link` cap (keeps the two parsers agreeing + // on where deep `[` stops nesting) + if (state.open_link_tag_depth >= MAX_INLINE_NESTING_DEPTH) { + consume_delimiter_as_text(state, '['); + return true; + } flush_text(state); ensure_paragraph(state); @@ -97,65 +107,69 @@ export const try_complete_link = (state: MdzStreamParserState, link_stack_idx: n return true; } - // found ]( — scan for ) - let i = bracket_pos + 2; // past ]( - while (i < state.buffer.length) { - const c = state.buffer.charCodeAt(i); - if (c === RIGHT_PAREN) { - const reference = state.buffer.slice(bracket_pos + 2, i); - // validate reference: non-empty and only valid path chars - if (!reference.trim()) { - abort_link_to_text(state, link_stack_idx, bracket_pos); - return true; - } - let valid = true; - for (let j = 0; j < reference.length; j++) { - if (!is_valid_path_char(reference.charCodeAt(j))) { - valid = false; - break; - } - } - if (!valid) { - abort_link_to_text(state, link_stack_idx, bracket_pos); - return true; - } - // Reject unsafe protocols (javascript:, data:, etc.) at parse time. - // `is_valid_path_char` permits `:` so we have to filter explicitly. - if (!mdz_is_safe_reference(reference)) { - abort_link_to_text(state, link_stack_idx, bracket_pos); - return true; - } - - // success — close the link with reference - flush_text(state); - revert_above(state, link_stack_idx); - const entry = pop_stack_entry(state); - const link_type = mdz_is_url(reference) ? 'external' : 'internal'; - emit(state, { - type: 'close', - id: entry.id, - end: offset(state, i + 1), - reference, - link_type, - }); - state.active_text_id = null; - state.pos = i + 1; - state.column += i + 1 - bracket_pos; - state.prev_char = RIGHT_PAREN; + // found ]( — find the reference's terminator: `)` completes, a space or + // newline aborts. Memoized searches (see `buffer_index_of`) — a link whose + // URL arrives incrementally re-enters here on every feed with `pos` still + // at the `]`, and a raw scan would re-cover the growing reference each + // time, going quadratic on long streamed URLs. + const ref_start = bracket_pos + 2; + const close_pos = buffer_index_of(state, ')', ref_start); + const space_pos = buffer_index_of(state, ' ', ref_start); + const newline_pos = buffer_index_of(state, '\n', ref_start); + let invalid_pos = space_pos; + if (newline_pos !== -1 && (invalid_pos === -1 || newline_pos < invalid_pos)) { + invalid_pos = newline_pos; + } + + if (close_pos === -1) { + // didn't find ) — need more input or invalid + if (invalid_pos === -1) return false; // need more input + // found invalid char before any ) — revert + abort_link_to_text(state, link_stack_idx, bracket_pos); + return true; + } + if (invalid_pos !== -1 && invalid_pos < close_pos) { + // invalid character in URL (simplified check) + abort_link_to_text(state, link_stack_idx, bracket_pos); + return true; + } + + const i = close_pos; + const reference = state.buffer.slice(ref_start, i); + // validate reference: non-empty and only valid path chars + if (!reference.trim()) { + abort_link_to_text(state, link_stack_idx, bracket_pos); + return true; + } + for (let j = 0; j < reference.length; j++) { + if (!is_valid_path_char(reference.charCodeAt(j))) { + abort_link_to_text(state, link_stack_idx, bracket_pos); return true; } - if (c === NEWLINE || c === SPACE) { - // invalid character in URL (simplified check) - break; - } - i++; + } + // Reject unsafe protocols (javascript:, data:, etc.) at parse time. + // `is_valid_path_char` permits `:` so we have to filter explicitly. + if (!mdz_is_safe_reference(reference)) { + abort_link_to_text(state, link_stack_idx, bracket_pos); + return true; } - // didn't find ) — need more input or invalid - if (i >= state.buffer.length) return false; // need more input - - // found invalid char before ) — revert - abort_link_to_text(state, link_stack_idx, bracket_pos); + // success — close the link with reference + flush_text(state); + revert_above(state, link_stack_idx); + const entry = pop_stack_entry(state); + const link_type = mdz_is_url(reference) ? 'external' : 'internal'; + emit(state, { + type: 'close', + id: entry.id, + end: offset(state, i + 1), + reference, + link_type, + }); + state.active_text_id = null; + state.pos = i + 1; + state.column += i + 1 - bracket_pos; + state.prev_char = RIGHT_PAREN; return true; }; @@ -176,7 +190,19 @@ export const try_tag_open = (state: MdzStreamParserState): TryResult => { return 'not_match'; } + // nesting-depth cap: `` closer still closes; a `<` at + // the buffer end still holds), so only genuine opens are suppressed. + if (state.open_link_tag_depth >= MAX_INLINE_NESTING_DEPTH) { + consume_delimiter_as_text(state, '<'); + return 'consumed'; + } + // collect tag name + // TODO the name/whitespace scans rescan from `start` on every held retry + // of an unclosed tag's open tail — needs a hold watermark; a length cap would + // diverge from the sync parser on pathological multi-KB "tag names" const name_start = i; while (i < state.buffer.length && is_tag_name_char(state.buffer.charCodeAt(i))) { i++; @@ -255,9 +281,16 @@ export const try_close_tag = (state: MdzStreamParserState): TryResult => { const name = state.buffer.slice(name_start, i); i++; // past > + // O(1) bail when no tag of this name is open — without it, every `` + // candidate walks the whole inline stack, which grows one frame per leaked + // unclosed `` and goes quadratic on tag-dense input (the same shape + // `open_counts` guards for the delimiter inlines) + if (!state.open_tag_counts?.get(name)) return 'not_match'; + // find matching open tag on stack let found_idx = -1; for (let j = state.stack.length - 1; j >= 0; j--) { + if (mdz_debug_work.enabled) mdz_debug_work.total++; // frames walked (mismatched-tags guard) const entry = state.stack[j]!; if ( (entry.node_type === 'Element' || entry.node_type === 'Component') && diff --git a/src/lib/mdz_stream_parser_list.ts b/src/lib/mdz_stream_parser_list.ts index e77037f..675047d 100644 --- a/src/lib/mdz_stream_parser_list.ts +++ b/src/lib/mdz_stream_parser_list.ts @@ -418,10 +418,16 @@ export const process_list_line = ( if (marker !== null && marker.empty) { // empty item: only at an indent exactly matching an open level of the - // same marker type — anything else degrades below - const level_idx = state.list_levels.findIndex( - (l) => l.indent === indent && l.ordered === marker.ordered, - ); + // same marker type — anything else degrades below (manual loop like + // `list_continuation_target` — no per-line closure allocation) + let level_idx = -1; + for (let li = 0; li < state.list_levels.length; li++) { + const l = state.list_levels[li]!; + if (l.indent === indent && l.ordered === marker.ordered) { + level_idx = li; + break; + } + } if (level_idx !== -1) { close_list_run(state); while (state.list_levels.length - 1 > level_idx) { diff --git a/src/lib/mdz_stream_parser_state.ts b/src/lib/mdz_stream_parser_state.ts index 314336e..ab7da0f 100644 --- a/src/lib/mdz_stream_parser_state.ts +++ b/src/lib/mdz_stream_parser_state.ts @@ -9,7 +9,9 @@ */ import type {MdzNodeTypeContainer, MdzNodeId, MdzOpcode} from './mdz_opcodes.ts'; +import type {MdzTableCellParser} from './mdz.ts'; import {NEWLINE, has_non_whitespace, mdz_heading_id_from_text} from './mdz_helpers.ts'; +import {mdz_debug_work} from './mdz_debug_work.ts'; /** * Tri-state result for `try_*` parser handlers. @@ -214,8 +216,9 @@ export interface MdzStreamParserState { * is `null` for a top-level table (column-0 rows) or the enclosing list item's * marker indent for a table nested as a block child (rows skip the indent and * a dedent to that indent ends the table, returning control to the list). + * `cell_parser` is the table's shared cell parser, rebound per row. */ - table: {id: MdzNodeId; item_indent: number | null} | null; + table: {id: MdzNodeId; item_indent: number | null; cell_parser: MdzTableCellParser} | null; /** * Whether the innermost open ListItem's first inline run is current. * Content then flows directly into the ListItem frame — the sync AST @@ -243,6 +246,32 @@ export interface MdzStreamParserState { search_memo: Map | null; /** Open-container counts for `find_open`'s O(1) "nothing open" early exit. */ open_counts: OpenInlineCounts; + /** + * Count of open `Link`/`Element`/`Component` frames in the current run — the + * streaming analogue of the sync lexer's `#inline_depth`. Kept as an O(1) + * counter (not a stack walk, which would reintroduce the deep-stack + * quadratic `open_counts` exists to avoid) so `try_link_open`/`try_tag_open` + * can cap nesting at `MAX_INLINE_NESTING_DEPTH` — rendering a deeper `[`/`<` + * literal, matching the sync cap on pure/symmetric nesting. It counts + * **optimistic** opens, though, so it diverges from the sync `#inline_depth` + * (which the sync tag path only bumps after a closer precheck) on an + * asymmetric cap-saturating prefix — 100+ *unclosed* `` before a valid + * `[x](/u)` saturates this counter and suppresses the trailing link that sync + * still forms. Adversarial-only, documented not fixed (streaming can't + * precheck closers): see the flip-at-cap case in `mdz_nesting_cap.test.ts`. + * Run-scoped for free like `open_counts`: a + * run close reverts every inline frame above it, decrementing back to zero. + */ + open_link_tag_depth: number; + /** + * Per-name counts of open Element/Component frames, the tag analogue of + * `open_counts`: `try_close_tag` bails in O(1) when the closing name isn't + * open, instead of walking a stack that grows one frame per leaked unclosed + * ``. Lazily created on the first tag open — tag-free inputs never + * allocate the map. Run-scoped like `open_counts`: run closes revert every + * inline frame above them, decrementing the counts back to zero. + */ + open_tag_counts: Map | null; /** Whether we're inside a heading (newline ends it). */ in_heading: boolean; /** Whether we're inside an optimistic inline Code container. */ @@ -315,6 +344,8 @@ export const create_state = (): MdzStreamParserState => ({ base_offset: 0, search_memo: null, open_counts: {Bold: 0, Italic: 0, Strikethrough: 0, Link: 0}, + open_link_tag_depth: 0, + open_tag_counts: null, in_heading: false, in_code: false, in_paragraph: false, @@ -362,6 +393,13 @@ export const push_stack_entry = ( } else { bump_open_count(state, node_type, 1); } + if (tag_name !== undefined) { + const counts = (state.open_tag_counts ??= new Map()); + counts.set(tag_name, (counts.get(tag_name) ?? 0) + 1); + } + // Link/Element/Component contribute to the nesting-depth cap (tag_name is + // set for Element/Component only, so `Link || tag_name` is exactly the three) + if (node_type === 'Link' || tag_name !== undefined) state.open_link_tag_depth++; }; /** @@ -394,6 +432,16 @@ const bump_open_count = ( export const pop_stack_entry = (state: MdzStreamParserState): StackEntry => { const entry = state.stack.pop()!; bump_open_count(state, entry.node_type, -1); + if (entry.tag_name !== undefined) { + const counts = state.open_tag_counts!; + // drop zero-count keys so a long stream of many distinct tag names can't + // grow the map unboundedly — the `!counts.get(name)` bail in `try_close_tag` + // treats a missing key and a 0 count identically, so this is behavior-neutral + const remaining = counts.get(entry.tag_name)! - 1; + if (remaining === 0) counts.delete(entry.tag_name); + else counts.set(entry.tag_name, remaining); + } + if (entry.node_type === 'Link' || entry.tag_name !== undefined) state.open_link_tag_depth--; return entry; }; @@ -465,7 +513,11 @@ const memo_index_of = ( // rescan only the appended tail, overlapping the old boundary so a // straddling needle isn't missed const rescan_from = Math.max(global_from, memo.searched_to - (needle.length - 1)); - const result = state.buffer.indexOf(needle, rescan_from - state.base_offset); + const local_from = rescan_from - state.base_offset; + const result = state.buffer.indexOf(needle, local_from); + if (mdz_debug_work.enabled) { + mdz_debug_work.total += (result === -1 ? state.buffer.length : result) - local_from; + } memo.from = global_from; memo.result = result === -1 ? -1 : state.base_offset + result; memo.searched_to = global_end; @@ -474,6 +526,9 @@ const memo_index_of = ( // found result already behind `from` — fall through to a fresh scan } const result = state.buffer.indexOf(needle, from); + if (mdz_debug_work.enabled) { + mdz_debug_work.total += (result === -1 ? state.buffer.length : result) - from; + } memo.from = global_from; memo.result = result === -1 ? -1 : state.base_offset + result; memo.searched_to = global_end; @@ -519,8 +574,13 @@ export const emit = (state: MdzStreamParserState, op: MdzOpcode): void => { if (top) { top.has_children = true; // propagate non-whitespace marker to the nearest Paragraph so - // whitespace-only paragraphs can be discarded at close - if (op.type === 'void' || has_non_whitespace(op.content)) { + // whitespace-only paragraphs can be discarded at close — the O(1) + // no-paragraph check runs first so codeblock/table-cell emissions + // (never inside a Paragraph) skip the content scan entirely + if ( + op.type === 'void' || + (state.paragraph_stack_idx !== -1 && has_non_whitespace(op.content)) + ) { mark_paragraph_non_whitespace(state); } } @@ -744,6 +804,18 @@ export const revert_failed_close = (state: MdzStreamParserState, stack_idx: numb const frames_above = state.stack.length - 1 - stack_idx; state.stack.splice(stack_idx, 1); bump_open_count(state, entry.node_type, -1); + if (entry.tag_name !== undefined) { + const counts = state.open_tag_counts!; + // drop zero-count keys so a long stream of many distinct tag names can't + // grow the map unboundedly — the `!counts.get(name)` bail in `try_close_tag` + // treats a missing key and a 0 count identically, so this is behavior-neutral + const remaining = counts.get(entry.tag_name)! - 1; + if (remaining === 0) counts.delete(entry.tag_name); + else counts.set(entry.tag_name, remaining); + } + // only ever reverts an Italic today, but keep the depth counter honest if a + // Link/Element/Component is ever failed-closed mid-stack + if (entry.node_type === 'Link' || entry.tag_name !== undefined) state.open_link_tag_depth--; state.opcodes.push({ type: 'revert', id: entry.id, diff --git a/src/lib/mdz_stream_parser_table.ts b/src/lib/mdz_stream_parser_table.ts index 4704f27..09a9c77 100644 --- a/src/lib/mdz_stream_parser_table.ts +++ b/src/lib/mdz_stream_parser_table.ts @@ -7,8 +7,9 @@ * whole header emits at once. Body rows then stream one per line. Because a row * is always fully buffered before it emits (no within-row streaming), each * cell's inline content is parsed with the sync reference (`MdzTableCellParser`, - * one `MdzLexer`/`MdzTokenParser` shared across the row's cells) and replayed - * as opcodes — guaranteeing sync↔stream parity by construction. + * one `MdzLexer`/`MdzTokenParser` shared across the whole table's cells, + * rebound per row) and replayed as opcodes — guaranteeing sync↔stream parity + * by construction. * * @module */ @@ -72,7 +73,11 @@ export const try_table_start = (state: MdzStreamParserState, forced: boolean): T const table_start = offset(state, start); emit(state, {type: 'open', id: table_id, node_type: 'Table', start: table_start, align}); push_stack_entry(state, table_id, 'Table', table_start); - state.table = {id: table_id, item_indent: null}; + state.table = { + id: table_id, + item_indent: null, + cell_parser: new MdzTableCellParser(state.buffer), + }; emit_table_row(state, header_cells, true, start, header_end); @@ -280,7 +285,11 @@ export const open_table_in_item = ( align: head.align, }); push_stack_entry(state, table_id, 'Table', table_start); - state.table = {id: table_id, item_indent: marker_indent}; + state.table = { + id: table_id, + item_indent: marker_indent, + cell_parser: new MdzTableCellParser(state.buffer), + }; emit_table_row(state, head.header_cells, true, header_first, head.header_end); // leave the cursor on the delimiter's newline (or EOF) — the in-item body-row // handler reads from the line after it and hands back on this same convention @@ -301,9 +310,11 @@ const emit_table_row = ( const row_off = offset(state, row_start); emit(state, {type: 'open', id: row_id, node_type: 'TableRow', start: row_off, header}); push_stack_entry(state, row_id, 'TableRow', row_off); - // one lexer/parser shared across the row's cells (all index into the same - // buffer) rather than allocated per cell - const cell_parser = new MdzTableCellParser(state.buffer); + // one lexer/parser shared across the whole table rather than allocated per + // row (or per cell) — rebound per row because the buffer string changes + // across feeds; the lexer's search memo survives when it doesn't + const cell_parser = state.table!.cell_parser; + cell_parser.rebind(state.buffer); for (const cell of cells) { emit_table_cell(state, cell_parser, cell.start, cell.end); } diff --git a/src/lib/mdz_stream_parser_text.ts b/src/lib/mdz_stream_parser_text.ts index 6046d34..267e1d6 100644 --- a/src/lib/mdz_stream_parser_text.ts +++ b/src/lib/mdz_stream_parser_text.ts @@ -5,21 +5,15 @@ */ import { - ASTERISK, - BACKTICK, HTTPS_PREFIX_LENGTH, - H_LOWER, - H_UPPER, - LEFT_ANGLE, - LEFT_BRACKET, NEWLINE, - PERIOD, - RIGHT_BRACKET, - SLASH, SPACE, TAB, - TILDE, - UNDERSCORE, + TEXT_CLASS_NEWLINE, + TEXT_CLASS_PATH, + TEXT_CLASS_STOP, + TEXT_CLASS_URL, + TEXT_SCAN_CLASS, is_word_char, match_url_prefix_case_insensitive, } from './mdz_helpers.ts'; @@ -29,6 +23,7 @@ import { ensure_paragraph, offset, } from './mdz_stream_parser_state.ts'; +import {mdz_debug_work} from './mdz_debug_work.ts'; /** * Consume a delimiter at the current position as literal text — the shared @@ -60,35 +55,41 @@ export const consume_text_run = (state: MdzStreamParserState): void => { state.pos++; while (state.pos < state.buffer.length) { const c = state.buffer.charCodeAt(state.pos); - if ( - c === BACKTICK || - c === ASTERISK || - c === UNDERSCORE || - c === TILDE || - c === LEFT_BRACKET || - c === RIGHT_BRACKET || - c === LEFT_ANGLE || - c === NEWLINE || - // URL/path: only break when actually at a URL or path start. - // Most h/H/./slash in prose are not URLs/paths — continuing the scan - // avoids the full dispatch cycle through process_loop → process_inline. - // For `h`/`H` at a word boundary, also break when there isn't enough - // buffer left to confirm `https://` / `http://` — process_inline's - // speculator then carries the prefix match across chunk boundaries. - // Scheme matching is case-insensitive (RFC 3986). - ((c === H_LOWER || c === H_UPPER) && - (match_url_prefix_case_insensitive(state.buffer, state.pos) > 0 || - (state.buffer.length - state.pos < HTTPS_PREFIX_LENGTH && - !is_word_char(state.buffer.charCodeAt(state.pos - 1))))) || - ((c === SLASH || c === PERIOD) && - (state.buffer.charCodeAt(state.pos - 1) === SPACE || - state.buffer.charCodeAt(state.pos - 1) === NEWLINE || - state.buffer.charCodeAt(state.pos - 1) === TAB)) - ) { - break; + // one table load classifies the char; zero (the common case) continues + // the run without any further checks. `TEXT_SCAN_CLASS` is shared with + // the sync lexer's `#tokenize_text` (`mdz_lexer.ts`); the dispatch bodies + // intentionally differ — that one handles `\|` escapes and list-run + // strip, this one URL speculation across chunk boundaries. + const char_class = c < 128 ? TEXT_SCAN_CLASS[c]! : 0; + if (char_class === 0) { + state.pos++; + continue; } + if ((char_class & (TEXT_CLASS_STOP | TEXT_CLASS_NEWLINE)) !== 0) break; + if ((char_class & TEXT_CLASS_URL) !== 0) { + // URL: only break when actually at a URL start. Most h/H in prose are + // not URLs — continuing the scan avoids the full dispatch cycle + // through process_loop → process_inline. At a word boundary, also + // break when there isn't enough buffer left to confirm `https://` / + // `http://` — process_inline's speculator then carries the prefix + // match across chunk boundaries. Scheme matching is case-insensitive + // (RFC 3986). + if ( + match_url_prefix_case_insensitive(state.buffer, state.pos) > 0 || + (state.buffer.length - state.pos < HTTPS_PREFIX_LENGTH && + !is_word_char(state.buffer.charCodeAt(state.pos - 1))) + ) { + break; + } + } else if ((char_class & TEXT_CLASS_PATH) !== 0) { + // path (`/` or `.`): only break at a plausible path start + const prev = state.buffer.charCodeAt(state.pos - 1); + if (prev === SPACE || prev === NEWLINE || prev === TAB) break; + } + // backslash (or a non-starting path char) — ordinary text state.pos++; } + if (mdz_debug_work.enabled) mdz_debug_work.total += state.pos - start; accumulate_text(state, state.buffer.slice(start, state.pos), offset(state, start)); state.prev_char = state.buffer.charCodeAt(state.pos - 1); state.column += state.pos - start; diff --git a/src/lib/mdz_to_svelte.ts b/src/lib/mdz_to_svelte.ts index 5d166b9..3f43550 100644 --- a/src/lib/mdz_to_svelte.ts +++ b/src/lib/mdz_to_svelte.ts @@ -13,7 +13,7 @@ import {escape_svelte_text} from '@fuzdev/fuz_util/svelte_preprocess_helpers.ts' import {escape_js_string} from '@fuzdev/fuz_util/string.ts'; import type {MdzNode, MdzNodeTableRow} from './mdz.ts'; -import {mdz_resolve_relative_path, mdz_is_void_element} from './mdz_helpers.ts'; +import {mdz_is_void_element, mdz_classify_link} from './mdz_helpers.ts'; /** * Result of converting `MdzNode` arrays to Svelte markup. @@ -131,26 +131,21 @@ export const mdz_to_svelte = ( case 'Link': { const children_markup = render_nodes(node.children); - if (node.link_type === 'internal') { - if (node.reference.startsWith('.') && base) { - const resolved = mdz_resolve_relative_path(node.reference, base); - imports.set('resolve', {path: '$app/paths', kind: 'named'}); - return `${children_markup}`; - } - if ( - node.reference.startsWith('#') || - node.reference.startsWith('?') || - node.reference.startsWith('.') || - // Bare references (e.g. `foo`) aren't absolute — resolve() throws on them - !node.reference.startsWith('/') - ) { - return `${children_markup}`; - } + // safety gate + resolve-vs-raw classification shared with the runtime + // views via `mdz_classify_link`, so the three renderers can't drift + // (an unsafe `javascript:`/`data:` ref renders as plain children, no + // `` — defense in depth, since `mdz_to_svelte` is a public entry + // callable with hand-built trees that never went through `mdz_parse`) + const link = mdz_classify_link(node.reference, node.link_type, base); + if (link.kind === 'unsafe') return children_markup; + if (link.kind === 'resolve') { imports.set('resolve', {path: '$app/paths', kind: 'named'}); - return `${children_markup}`; + return `${children_markup}`; + } + if (link.kind === 'external') { + return `${children_markup}`; } - // External link — matches MdzNodeView: target="_blank" rel="noopener" - return `${children_markup}`; + return `${children_markup}`; } case 'Paragraph': @@ -184,19 +179,16 @@ export const mdz_to_svelte = ( } return `${cells}`; }; + // by grammar the header row is always first (and the only one) — + // read it by index and skip it in the body, mirroring the node + // views (a `.filter()` pair re-partitions every row; see + // `MdzNodeView.svelte`'s "Table rendering de-quadratified" note) let out = ''; - const header_rows = node.children.filter((r) => r.header); - const body_rows = node.children.filter((r) => !r.header); - if (header_rows.length > 0) { - out += ''; - for (const row of header_rows) out += render_row(row, 'th'); - out += ''; - } - if (body_rows.length > 0) { - out += ''; - for (const row of body_rows) out += render_row(row, 'td'); - out += ''; - } + const header_row = node.children[0]?.header ? node.children[0] : undefined; + if (header_row) out += '' + render_row(header_row, 'th') + ''; + let body = ''; + for (const row of node.children) if (!row.header) body += render_row(row, 'td'); + if (body) out += '' + body + ''; return out + '
    '; } diff --git a/src/lib/mdz_token_parser.ts b/src/lib/mdz_token_parser.ts index 9b05399..e6b2c66 100644 --- a/src/lib/mdz_token_parser.ts +++ b/src/lib/mdz_token_parser.ts @@ -46,6 +46,21 @@ import { type MdzTokenTableCellOpen, } from './mdz_lexer.ts'; +/** + * Whether a token type ends the current inline context — the block-boundary + * set shared by the delimiter-pair and link children loops (an unclosed + * inline never swallows block structure). + */ +const token_is_block_boundary = (type: MdzToken['type']): boolean => + type === 'paragraph_break' || + type === 'heading_start' || + type === 'hr' || + type === 'codeblock' || + type === 'list_open' || + type === 'list_item_close' || + type === 'blockquote_open' || + type === 'table_open'; + /** * Builds an `MdzNode[]` tree from a lexed `MdzToken[]` stream. * Used by `mdz_parse`, which should be preferred for simple usage. @@ -490,16 +505,7 @@ export class MdzTokenParser { this.#index++; return {type: node_type, children, start, end: t.end}; } - if ( - t.type === 'paragraph_break' || - t.type === 'heading_start' || - t.type === 'hr' || - t.type === 'codeblock' || - t.type === 'list_open' || - t.type === 'list_item_close' || - t.type === 'blockquote_open' || - t.type === 'table_open' - ) { + if (token_is_block_boundary(t.type)) { break; } const node = this.#parse_inline(); @@ -539,16 +545,7 @@ export class MdzTokenParser { // No link_ref - treat as text return {type: 'Text', content: '[', start, end: start + 1}; } - if ( - t.type === 'paragraph_break' || - t.type === 'heading_start' || - t.type === 'hr' || - t.type === 'codeblock' || - t.type === 'list_open' || - t.type === 'list_item_close' || - t.type === 'blockquote_open' || - t.type === 'table_open' - ) { + if (token_is_block_boundary(t.type)) { break; } const node = this.#parse_inline(); diff --git a/src/lib/svelte_preprocess_mdz.ts b/src/lib/svelte_preprocess_mdz.ts index 5a548e3..a6693d9 100644 --- a/src/lib/svelte_preprocess_mdz.ts +++ b/src/lib/svelte_preprocess_mdz.ts @@ -37,7 +37,7 @@ import { } from '@fuzdev/fuz_util/svelte_preprocess_helpers.ts'; import {mdz_parse} from './mdz.ts'; -import {mdz_to_svelte} from './mdz_to_svelte.ts'; +import {mdz_to_svelte, type MdzToSvelteResult} from './mdz_to_svelte.ts'; /** * An estree `ImportDeclaration` augmented with Svelte's position data. @@ -296,6 +296,33 @@ const collect_consumed_bindings = ( return consumed; }; +/** + * Parse a static mdz string and render it to Svelte markup. Returns `null` to + * leave the usage for the runtime — either the content raised a parse error + * (reported via `on_error`) or it contains unconfigured tags (which the + * precompiler emits nothing for; the runtime renders the fallback chrome). + */ +const render_static_mdz = ( + content: string, + base: string | undefined, + context: FindMdzUsagesContext, +): MdzToSvelteResult | null => { + let result: MdzToSvelteResult; + try { + result = mdz_to_svelte(mdz_parse(content), { + components: context.components, + elements: context.elements, + base, + code_component_import: context.code_component_import, + codeblock_component_import: context.codeblock_component_import, + }); + } catch (error) { + handle_preprocess_error(error, '[fuz-mdz]', context.filename, context.on_error); + return null; + } + return result.has_unconfigured_tags ? null : result; +}; + /** * Walks the AST to find `Mdz` component usages with static `content` props * and generates transformations to replace them with `MdzPrecompiled` children. @@ -346,24 +373,8 @@ const find_mdz_usages = ( // Extract static string value const content_value = extract_static_string(content_attr.value, context.bindings); if (content_value !== null) { - // Parse mdz content and render to Svelte markup - let result; - try { - const nodes = mdz_parse(content_value); - result = mdz_to_svelte(nodes, { - components: context.components, - elements: context.elements, - base, - code_component_import: context.code_component_import, - codeblock_component_import: context.codeblock_component_import, - }); - } catch (error) { - handle_preprocess_error(error, '[fuz-mdz]', context.filename, context.on_error); - return; - } - - // If content has unconfigured tags, skip this usage (fall back to runtime) - if (result.has_unconfigured_tags) return; + const result = render_static_mdz(content_value, base, context); + if (result === null) return; // parse error or unconfigured tags — leave for runtime const consumed = collect_consumed_bindings(content_attr.value, context.bindings); const replacement = build_replacement(node, exclude_attrs, result.markup, context.source); @@ -390,22 +401,10 @@ const find_mdz_usages = ( // Parse and render each branch const branch_results: Array<{markup: string; imports: Map}> = []; - try { - for (const branch of chain) { - const nodes = mdz_parse(branch.value); - const result = mdz_to_svelte(nodes, { - components: context.components, - elements: context.elements, - base, - code_component_import: context.code_component_import, - codeblock_component_import: context.codeblock_component_import, - }); - if (result.has_unconfigured_tags) return; - branch_results.push({markup: result.markup, imports: result.imports}); - } - } catch (error) { - handle_preprocess_error(error, '[fuz-mdz]', context.filename, context.on_error); - return; + for (const branch of chain) { + const result = render_static_mdz(branch.value, base, context); + if (result === null) return; // parse error or unconfigured tags — leave for runtime + branch_results.push({markup: result.markup, imports: result.imports}); } // Build {#if}/{:else if}/{:else} markup diff --git a/src/routes/docs/streaming/mdz_streaming_paths.mdz b/src/routes/docs/streaming/mdz_streaming_paths.mdz index 58a0179..a33d5c8 100644 --- a/src/routes/docs/streaming/mdz_streaming_paths.mdz +++ b/src/routes/docs/streaming/mdz_streaming_paths.mdz @@ -8,7 +8,7 @@ For inline use in a Svelte template, with content known up front: ``` -Internally calls `mdz_parse` and renders via `MdzNodeView`. Best for: documentation pages, alerts, tooltips — anything where you have the full string before render. With /docs/svelte_preprocess_mdz this also compiles away at build time for static strings. +Internally calls `mdz_parse` and renders via `MdzNodeView`. Best for: documentation pages, alerts, tooltips — anything where you have the full string before render. With /docs/svelte_preprocess_mdz this also compiles away at build time for static strings. To skip the parse, pass a pre-parsed tree instead of a string — `` — for content parsed ahead of render (e.g. a docs site pre-parsing its markdown at build time). ### Path 2: `mdz_parse` + `MdzNodeView` (one-shot, manual) diff --git a/src/routes/docs/usage/grammar/+page.svelte b/src/routes/docs/usage/grammar/+page.svelte index 64e62cc..782132b 100644 --- a/src/routes/docs/usage/grammar/+page.svelte +++ b/src/routes/docs/usage/grammar/+page.svelte @@ -198,6 +198,14 @@ any whitespace invalidates the link + + inline nesting depth + no depth limit + link/tag containers cap out — deeply nested [/<tag> render + literal (a strict untrusted-content bound) + hard breaks trailing double-space or \ diff --git a/src/routes/docs/usage/grammar/mdz_grammar.mdz b/src/routes/docs/usage/grammar/mdz_grammar.mdz index 23a7338..1f09ca9 100644 --- a/src/routes/docs/usage/grammar/mdz_grammar.mdz +++ b/src/routes/docs/usage/grammar/mdz_grammar.mdz @@ -351,6 +351,8 @@ A non-self-closing tag commits only if its exact closing tag exists later in the **Tag scoping.** A tag's children are a bounded region, like a heading line or a list-item run: an inline delimiter inside the tag never pairs across the closing tag (in ```ax` ``, the backtick is literal — a code span can't swallow ``), and a nested same-name tag consumes the nearest closer, leaving the outer tag to find its own later (`xy` nests; when no closer remains for the outer tag, it degrades to literal `<` and the rest re-parses). +**Inline nesting is bounded.** Link and tag containers nest only so deep — past a fixed cap, a further `[` or `` renders as literal text instead of opening a new container. It's a strict untrusted-content bound (CommonMark permits such a limit), set deliberately far above any real content, so only pathological input reaches it. Delimiters (`**`, `_`, `~~`, `` ` ``) don't count toward the cap — first-closer-wins keeps them from deep-nesting on their own. + Components and elements must be registered (via `MdzRoot` or the contexts directly) to render; unregistered tags render as inert `` placeholders. See /docs/usage for the registration API. HTML void elements (`br`, `img`, `hr`, …) always render self-closing. The grammar still parses the open/close form (`
    x
    ` is an element with children — the parser doesn't know void-ness), but renderers drop the children, matching HTML, where a `
    ` closer is ignored. diff --git a/src/test/mdz_nesting_cap.test.ts b/src/test/mdz_nesting_cap.test.ts new file mode 100644 index 0000000..4aaff57 --- /dev/null +++ b/src/test/mdz_nesting_cap.test.ts @@ -0,0 +1,248 @@ +/** + * Guards for the inline link/tag nesting-depth cap and the adversarial + * backtracking cases it sits alongside (Bug 5's follow-up). + * + * Two concerns: + * + * 1. **Differential battery** — the sync lexer's link/tag failure paths still + * rewind and re-tokenize (the `#max_search_index` re-tokenization the memo + * kept), and its output on nested/mismatched `[`/`<` is load-bearing (it is + * the normative reference the streaming parser is checked against, and it + * diverges from the streaming parser on exactly these adversarial inputs). + * These summaries pin that output so a lexer refactor can't silently move it. + * None of these reach the depth cap. + * + * 2. **Nesting-depth cap** — beyond `MAX_INLINE_NESTING_DEPTH` open + * `Link`/`Element`/`Component` containers, a further `[`/`` renders + * literal in both parsers. This keeps the sync lexer linear and stack-safe + * on pathological nesting (see `mdz_scaling.test.ts`) and is mirrored in the + * streaming parser so the two agree on where deep nesting stops nesting. + */ + +import {test, assert, describe} from 'vitest'; + +import {mdz_parse, type MdzNode} from '$lib/mdz.ts'; +import {MAX_INLINE_NESTING_DEPTH} from '$lib/mdz_helpers.ts'; +import {MdzStreamParser} from '$lib/mdz_stream_parser.ts'; +import {mdz_opcodes_to_nodes} from '$lib/mdz_opcodes_to_nodes.ts'; + +/** Compact structural summary of a node tree — text/code inline, containers as `Type:tag(children)`. */ +const summarize = (nodes: Array): string => + nodes + .map((n): string => { + if (n.type === 'Text') return JSON.stringify(n.content); + if (n.type === 'Code') return '`' + n.content + '`'; + if (n.type === 'Codeblock') return 'CB[' + JSON.stringify(n.content) + ']'; + const children = 'children' in n ? '(' + summarize(n.children) + ')' : ''; + const tag = 'name' in n ? n.name : 'reference' in n ? n.reference : undefined; + return n.type + (tag !== undefined ? ':' + tag : '') + children; + }) + .join(' '); + +const stream_parse = (text: string): Array => { + const p = new MdzStreamParser(); + p.feed(text); + p.finish(); + return mdz_opcodes_to_nodes(p.take_opcodes()); +}; + +const stream_parse_chunked = (text: string, chunk: number): Array => { + const p = new MdzStreamParser(); + for (let i = 0; i < text.length; i += chunk) p.feed(text.slice(i, i + chunk)); + p.finish(); + return mdz_opcodes_to_nodes(p.take_opcodes()); +}; + +/** Whether the tree contains a node of the given type at any depth. */ +const contains_type = (nodes: Array, type: string): boolean => + nodes.some((n) => n.type === type || ('children' in n && contains_type(n.children, type))); + +/** Deepest `Element`/`Component`/`Link` nesting in a tree (0 for none). */ +const max_container_depth = (nodes: Array): number => { + let max = 0; + for (const n of nodes) { + if (!('children' in n)) continue; + const inner = max_container_depth(n.children); + const here = n.type === 'Element' || n.type === 'Component' || n.type === 'Link' ? 1 : 0; + max = Math.max(max, here + inner); + } + return max; +}; + +// The sync parser is the normative reference; these lock its output on the +// adversarial cases where its rewind-and-retokenize backtracking is +// observable. Regenerate the right-hand strings only with a deliberate, +// reviewed semantics change (never to make a refactor "pass"). +const BATTERY: Array<[string, string]> = [ + // tags-with-delimiters — the `#max_search_index` re-tokenization: a failed + // outer tag re-tokenizes its children under the restored wider bound + ['_xy_z', 'Paragraph("_x" Element:a("y") "_z")'], + ['_xy_ z', 'Paragraph("" Italic("x" Element:a("y")) " z")'], + ['`ax`', 'Paragraph(Element:b("`a") "x`")'], + ['xy', 'Element:b("x" Element:b("y"))'], + ['**boldinnermore**', 'Element:a("**bold" Element:a("inner") "more**")'], + ['_x_', 'Paragraph(Element:a("_x") "_")'], + ['[x](/u)', 'Paragraph("" Link:/u("x"))'], + ['x', 'Paragraph(Element:a("x") "")'], + ['**ab**cd', 'Paragraph(Bold("ab") "cd")'], + // links with delimiters / nesting + ['[a `b ] c`](/u)', 'Paragraph(Link:/u("a " `b ] c`))'], + ['[a[b](/u)', 'Paragraph("[a" Link:/u("b"))'], + ['[x](/u)', 'Paragraph(Link:/u(Element:a("x")))'], + ['[[[nested]]](/u)', 'Paragraph("[[[nested]]](/u)")'], + // mixed link/tag precedence (sync's depth-first, the accepted stream divergence) + ['[b](/u)', 'Paragraph("" Link:/u("b"))'], + ['[b](/u)', 'Paragraph("[" Element:a("b](/u)"))'], + ['``x', 'Paragraph(`` "x")'], + // table cell delegates to the same sync lexer + [ + '| _xy_z | b |\n| --- | --- |\n| c | d |\n', + 'Table(TableRow(TableCell("_x" Element:a("y") "_z") TableCell("b")) TableRow(TableCell("c") TableCell("d")))', + ], +]; + +describe('adversarial nesting output is stable (Bug 5 differential battery)', () => { + for (const [input, expected] of BATTERY) { + test(JSON.stringify(input), () => { + assert.strictEqual(summarize(mdz_parse(input)), expected); + }); + } +}); + +describe('inline nesting-depth cap', () => { + const CAP = MAX_INLINE_NESTING_DEPTH; + + test('opens right up to the cap', () => { + // exactly CAP balanced tags nest fully — the cap is not yet reached + const input = ''.repeat(CAP) + 'X' + ''.repeat(CAP); + assert.strictEqual(max_container_depth(mdz_parse(input)), CAP); + }); + + test('the container one past the cap renders literal', () => { + // CAP+1 balanced tags: the deepest open would be at depth CAP, so it is + // suppressed and its `` becomes literal text — nesting stops at CAP + const input = ''.repeat(CAP + 1) + 'X' + ''.repeat(CAP + 1); + assert.strictEqual(max_container_depth(mdz_parse(input)), CAP); + }); + + test('deep unclosed [ stays literal, no throw', () => { + // far past the cap and past the old recursion ceiling — must not overflow + const nodes = mdz_parse('['.repeat(CAP * 5) + 'text'); + assert.isArray(nodes); + assert.strictEqual(max_container_depth(nodes), 0); // all literal + }); + + test('sync and streaming agree at and beyond the cap (pure nesting)', () => { + for (const n of [CAP - 1, CAP, CAP + 1, CAP + 2, CAP * 2]) { + for (const input of [ + '['.repeat(n) + 'text', + ''.repeat(n) + 'X' + ''.repeat(n), + ''.repeat(n) + 'X', + ]) { + assert.strictEqual( + JSON.stringify(mdz_parse(input)), + JSON.stringify(stream_parse(input)), + `parity at n=${n}: ${JSON.stringify(input.slice(0, 12))}…`, + ); + } + } + }); + + test('the cap is per-run — a nest past it does not suppress a later run', () => { + // depth resets when the paragraph closes: the second paragraph's link + // still forms even though the first blew past the cap + const input = '['.repeat(CAP + 50) + 'a\n\n[link](/u)'; + const nodes = mdz_parse(input); + assert.isTrue(contains_type(nodes, 'Link')); + assert.strictEqual(JSON.stringify(nodes), JSON.stringify(stream_parse(input))); + }); + + test('the cap applies (and resets) in every inline context', () => { + // heading, list item, table cell, and blockquote all reach inline + // tokenization through the same cap; a deep nest in each must not throw + // and must stay in sync/stream parity. The blockquote sub-lexes on a + // fresh lexer, so its depth is independent of any outer context. + const deep_bracket = '['.repeat(CAP + 20) + 'x'; + const deep_tag = ''.repeat(CAP + 20) + 'X'; + for (const input of [ + `# ${deep_bracket}`, + `- ${deep_tag}`, + `| ${deep_bracket} | b |\n| --- | --- |\n| c | d |\n`, + `> ${deep_bracket}\n\n[after](/u)`, + `> ${''.repeat(CAP)}X${''.repeat(CAP)}`, // exactly-cap nesting inside a quote + ]) { + const nodes = mdz_parse(input); + assert.isArray(nodes); + assert.strictEqual( + JSON.stringify(nodes), + JSON.stringify(stream_parse(input)), + `context parity: ${JSON.stringify(input.slice(0, 16))}…`, + ); + } + }); + + test('sync and streaming agree across the cap under chunked feeding', () => { + for (const n of [CAP - 1, CAP, CAP + 1, CAP + 2, 150]) { + for (const input of [ + '['.repeat(n) + 'text', + ''.repeat(n) + 'X' + ''.repeat(n), + '[** '.repeat(n) + 'x', // mixed link/delimiter chain (overflowed pre-cap) + ]) { + const expected = JSON.stringify(mdz_parse(input)); + for (const chunk of [1, 3, 7, 64]) { + assert.strictEqual( + JSON.stringify(stream_parse_chunked(input, chunk)), + expected, + `chunk=${chunk} n=${n}: ${JSON.stringify(input.slice(0, 12))}…`, + ); + } + } + } + }); +}); + +describe('cap-saturating asymmetric prefix — accepted sync↔stream divergence', () => { + // The cap counts OPTIMISTIC opens toward the streaming `open_link_tag_depth`, + // but the sync tag path prechecks for a closer before opening — so a run of + // CAP+ *unclosed* `` (or a `[** ` chain) saturates the streaming cap and + // suppresses a following valid `[x](/u)` that the sync oracle still forms. + // Adversarial-only (unreachable in real content); streaming can't precheck + // closers (append-only), so this is documented, not fixed. It refutes the + // stronger reading that "both parsers agree at the cap" — they agree on + // pure/symmetric nesting (above), not here. The flip is at exactly the cap. + const CAP = MAX_INLINE_NESTING_DEPTH; + + // each shape: a depth-saturating prefix of `n` opens, then a valid link + const shapes: Array<[string, (n: number) => string]> = [ + ['unclosed prefix', (n) => ''.repeat(n) + '[x](/u)'], + ['[** delimiter chain', (n) => '[** '.repeat(n) + '[x](/u)'], + ]; + + for (const [label, build] of shapes) { + test(`${label}: agreement flips at exactly the cap`, () => { + // below the cap both agree (streaming reverts the doomed opens, the + // trailing link forms in both) + const below = build(CAP - 1); + assert.strictEqual( + JSON.stringify(mdz_parse(below)), + JSON.stringify(stream_parse(below)), + `${label}: agree at n=${CAP - 1}`, + ); + // at the cap they diverge (streaming's optimistic opens saturate it) + const at = build(CAP); + assert.notStrictEqual( + JSON.stringify(mdz_parse(at)), + JSON.stringify(stream_parse(at)), + `${label}: diverge at n=${CAP}`, + ); + }); + } + + test('unclosed prefix: sync forms the trailing Link, streaming suppresses it', () => { + // the clean case — the whole prefix is unclosed tags (no internal links), + // so the only Link either parser could produce is the trailing `[x](/u)` + const input = ''.repeat(CAP) + '[x](/u)'; + assert.isTrue(contains_type(mdz_parse(input), 'Link'), 'sync (oracle) forms it'); + assert.isFalse(contains_type(stream_parse(input), 'Link'), 'streaming suppresses it'); + }); +}); diff --git a/src/test/mdz_render.stream_parity.test.ts b/src/test/mdz_render.stream_parity.test.ts new file mode 100644 index 0000000..d51607c --- /dev/null +++ b/src/test/mdz_render.stream_parity.test.ts @@ -0,0 +1,76 @@ +/** + * Binds the streaming renderer (`MdzStream`/`MdzStreamNodeView`) to the static + * renderer (`Mdz`/`MdzNodeView`) at the rendered-HTML level: every fixture is + * SSR'd through both and the normalized markup is diffed. Closes the gap that + * `MdzStreamNodeView` otherwise has no committed correctness test — tree parity + * (`mdz_parser_parity.test.ts`) binds opcode/tree output, not rendered HTML, so + * a markup-only divergence in the stream view (a missed attribute, wrong tag) + * would pass CI without this. + * + * Skipped: fixtures whose tree contains an `Element`/`Component` node. + * `MdzStreamState` deliberately does NOT apply the MDX single-tag-paragraph + * unwrap (`mdz_extract_single_tag`) that `mdz_parse`/`mdz_opcodes_to_nodes` do — + * it keeps per-id node identity for granular streaming updates — so a lone tag + * paragraph child stays `

    `-wrapped in the stream tree, a real HTML difference. + * The coarse skip over-skips (not every tag node is a sole paragraph child) but + * is simple and safe; the tag-rendering paths stay covered by the SSR corpus in + * `mdz_render.test.ts` and the tree-level parity suite. + */ + +import {describe, test, assert} from 'vitest'; +import {render} from 'svelte/server'; +import type {Component} from 'svelte'; + +import MdzComponent from '$lib/Mdz.svelte'; +import MdzStreamComponent from '$lib/MdzStream.svelte'; +import {MdzStreamParser} from '$lib/mdz_stream_parser.ts'; +import {MdzStreamState} from '$lib/mdz_stream_state.svelte.ts'; +import type {MdzNode} from '$lib/mdz.ts'; +import {load_fixtures} from './fixtures/mdz/mdz_test_helpers.ts'; + +const Mdz = MdzComponent as unknown as Component<{content: string}>; +const MdzStream = MdzStreamComponent as unknown as Component<{stream: MdzStreamState}>; + +/** Strip hydration comments and unwrap the `

    ` SSR wrapper. */ +const ssr_inner_html = (body: string): string => { + const stripped = body.replaceAll(//gs, ''); + assert.isTrue(stripped.startsWith('
    ') && stripped.endsWith('
    '), 'wrapper div'); + return stripped.slice('
    '.length, -'
    '.length); +}; + +const stream_ssr = (content: string): string => { + const parser = new MdzStreamParser(); + parser.feed(content); + parser.finish(); + const state = new MdzStreamState(); + state.apply_batch(parser.take_opcodes()); + return ssr_inner_html(render(MdzStream, {props: {stream: state}}).body); +}; + +/** See the module comment — the stream tree keeps lone tags `

    `-wrapped. */ +const has_tag_node = (nodes: Array): boolean => + nodes.some( + (n) => + n.type === 'Component' || + n.type === 'Element' || + ('children' in n && has_tag_node(n.children)), + ); + +describe('Mdz vs MdzStream SSR parity', () => { + test('every fixture renders identical HTML statically and streamed', async () => { + const fixtures = await load_fixtures(); + const failures: Array = []; + let compared = 0; + for (const fixture of fixtures) { + if (has_tag_node(fixture.expected)) continue; + const static_html = ssr_inner_html(render(Mdz, {props: {content: fixture.input}}).body); + const stream_html = stream_ssr(fixture.input); + if (static_html !== stream_html) { + failures.push(`${fixture.name}:\n static→ ${static_html}\n stream→ ${stream_html}`); + } + compared++; + } + assert.isAbove(compared, 400, 'compared fixture count'); + assert.deepEqual(failures.slice(0, 5), [], `${failures.length} of ${compared} diverge`); + }); +}); diff --git a/src/test/mdz_render.test.ts b/src/test/mdz_render.test.ts new file mode 100644 index 0000000..5fd7590 --- /dev/null +++ b/src/test/mdz_render.test.ts @@ -0,0 +1,36 @@ +/** + * SSR render tests for the runtime renderer (`Mdz` → `MdzNodeView`). + * + * The renderers previously had no test coverage at all — these bind the + * component output to the parser across the full fixture corpus via + * `svelte/server`'s `render` (no DOM environment needed). + */ + +import {describe, test, assert} from 'vitest'; +import {render} from 'svelte/server'; +import type {Component} from 'svelte'; + +import MdzComponent from '$lib/Mdz.svelte'; +import {load_fixtures} from './fixtures/mdz/mdz_test_helpers.ts'; + +// narrowed component type — the wrapper's `SvelteHTMLElements` rest-prop +// union is too complex for `render()`'s generic inference +const Mdz = MdzComponent as unknown as Component<{content: string}>; + +describe('Mdz SSR rendering', () => { + test('renders simple content', () => { + const result = render(Mdz, {props: {content: 'hello **bold** and `code`'}}); + assert.include(result.body, ''); + assert.include(result.body, 'bold'); + assert.include(result.body, 'code'); + }); + + test('renders every mdz fixture without throwing', async () => { + const fixtures = await load_fixtures(); + assert.isAbove(fixtures.length, 0); + for (const fixture of fixtures) { + const result = render(Mdz, {props: {content: fixture.input}}); + assert.isString(result.body, fixture.name); + } + }); +}); diff --git a/src/test/mdz_scaling.test.ts b/src/test/mdz_scaling.test.ts new file mode 100644 index 0000000..9e876a7 --- /dev/null +++ b/src/test/mdz_scaling.test.ts @@ -0,0 +1,90 @@ +/** + * Complexity regression guards for the parser's anti-quadratic / anti- + * exponential work — the one measurement class that's machine-state + * independent enough to gate CI (see the "Measurement robustness" note in the + * mdz perf lore). These are deliberately NOT wall-clock-threshold assertions + * (the dev machine swings ~2×); each input is sized so the fixed parser + * finishes in well under a millisecond while a reverted (super-linear) parser + * would blow past the vitest timeout by orders of magnitude. The timeout is + * the guard, so the huge margin makes them effectively unflakeable. + * + * Pair with the wall-clock scaling probe (`src/benchmarks/feed_scaling.local.ts`) + * and the benchmark suites for magnitude/ratio measurement; this file only + * answers linear-vs-not. + */ + +import {describe, test, assert} from 'vitest'; + +import {mdz_parse} from '$lib/mdz.ts'; +import {MdzStreamParser} from '$lib/mdz_stream_parser.ts'; + +const GUARD_TIMEOUT = 2000; + +/** Feed `content` in fixed-size chunks (exercises the streaming hold/compaction paths). */ +const feed_chunked = (content: string, chunk = 64): number => { + const parser = new MdzStreamParser(); + let count = 0; + for (let i = 0; i < content.length; i += chunk) { + parser.feed(content.slice(i, i + chunk)); + count += parser.take_opcodes().length; + } + parser.finish(); + count += parser.take_opcodes().length; + return count; +}; + +const feed_once = (content: string): number => { + const parser = new MdzStreamParser(); + parser.feed(content); + parser.finish(); + return parser.take_opcodes().length; +}; + +// Bug 5 — nested unclosed `[` / same-name `` in the sync lexer. The +// failure memo killed the 2ⁿ blowup; the `MAX_INLINE_NESTING_DEPTH` cap then +// made the sync path linear AND stack-safe (deeper opens render literal instead +// of recursing, bounding both the recursion depth and the number of reverting +// re-scans). At n=50_000 the fixed lexer runs in well under a second; any +// regression fails fast: dropping the cap stack-overflows (a thrown RangeError, +// not a hang), and dropping the memo re-descends exponentially — both overflow +// long before n=50_000. So this now guards time-complexity AND recursion depth, +// matching the streaming guards below (also n=50_000). The generous timeout is +// the linear-case ceiling, not the regression trigger. +const SYNC_DEEP_TIMEOUT = 8000; +describe('nested inline constructs are linear and stack-safe (Bug 5)', () => { + test('sync: nested unclosed [', {timeout: SYNC_DEEP_TIMEOUT}, () => { + assert.isArray(mdz_parse('['.repeat(50_000) + 'text')); + }); + test('sync: nested same-name ', {timeout: SYNC_DEEP_TIMEOUT}, () => { + assert.isArray(mdz_parse(''.repeat(50_000) + 'X')); + }); + test('sync: nested closed links to the cap and beyond', {timeout: SYNC_DEEP_TIMEOUT}, () => { + const n = 50_000; + assert.isArray(mdz_parse('['.repeat(n) + 'x' + ']'.repeat(n) + '(/u)')); + }); + test('streaming table cell (delegates to the sync lexer)', {timeout: SYNC_DEEP_TIMEOUT}, () => { + const table = `| ${'['.repeat(50_000)}X | b |\n| --- | --- |\n| c | d |\n`; + assert.isAbove(feed_once(table), 0); + }); +}); + +// Streaming held-candidate scans must stay linear under chunked feeds — the +// `buffer_index_of` / `open_tag_counts` memos. Inputs are large enough +// (128KB, fed 64B at a time) that a reverted (quadratic) scan would time out +// while the memoized scan stays in the low ms. +describe('streaming held-candidate scans stay linear', () => { + const BIG = 128_000; + test('hold line code (unclosed ` under open **)', {timeout: GUARD_TIMEOUT}, () => { + assert.isAbove(feed_chunked('**a `' + 'x'.repeat(BIG)), 0); + }); + test('hold line link (unterminated [text](url…)', {timeout: GUARD_TIMEOUT}, () => { + assert.isAbove(feed_chunked('[text](https://example.com/' + 'x'.repeat(BIG)), 0); + }); + test('mismatched tags (unclosed + never-matching closer)', {timeout: GUARD_TIMEOUT}, () => { + const input = Array.from({length: 4000}, (_, i) => `x`).join(''); + assert.isAbove(feed_once(input), 0); + }); + test('deep nesting stays linear (iterative streaming path)', {timeout: GUARD_TIMEOUT}, () => { + assert.isAbove(feed_once('['.repeat(50_000) + 'x'), 0); + }); +}); diff --git a/src/test/mdz_scaling_ratio.test.ts b/src/test/mdz_scaling_ratio.test.ts new file mode 100644 index 0000000..145d1a4 --- /dev/null +++ b/src/test/mdz_scaling_ratio.test.ts @@ -0,0 +1,126 @@ +/** + * Deterministic complexity-ratio guards for the parser hot paths. + * + * Unlike `mdz_scaling.test.ts` (wall-clock timeouts — machine-dependent, so the + * margins have to be huge), these assert on the `mdz_debug_work` scan-work + * counter: a pure function of input + algorithm, identical across machines/runs. + * For each pathology we parse at doubling input sizes and assert the work grows + * ~linearly with input length — a super-linear regression (a lost memo, a + * re-introduced rescan, a removed O(1) bail) makes the normalized ratio jump + * from ~1 (linear) toward ~2 (quadratic) for a 2× step, well clear of the + * threshold. + */ + +import {describe, test, assert} from 'vitest'; + +import {mdz_parse} from '$lib/mdz.ts'; +import {MdzStreamParser} from '$lib/mdz_stream_parser.ts'; +import {mdz_debug_work, mdz_debug_work_reset} from '$lib/mdz_debug_work.ts'; + +/** Scan work for a full sync parse. */ +const sync_work = (content: string): number => { + mdz_debug_work_reset(); + mdz_parse(content); + const total = mdz_debug_work.total; + mdz_debug_work_reset(false); + return total; +}; + +/** Scan work for a streaming parse, fed in `chunk`-byte slices (draining per feed). */ +const stream_work = (content: string, chunk: number): number => { + mdz_debug_work_reset(); + const p = new MdzStreamParser(); + if (chunk >= content.length) { + p.feed(content); + p.take_opcodes(); + } else { + for (let i = 0; i < content.length; i += chunk) { + p.feed(content.slice(i, i + chunk)); + p.take_opcodes(); + } + } + p.finish(); + p.take_opcodes(); + const total = mdz_debug_work.total; + mdz_debug_work_reset(false); + return total; +}; + +/** + * Assert scan work grows sub-quadratically across increasing input sizes. + * `normalized = work_ratio / size_ratio` is ~1 for linear, ~size_ratio (~2 for a + * doubling) for quadratic; `MAX_NORMALIZED` sits between with margin for the + * fixed-constant slop a linear algorithm carries at small sizes. + */ +const MAX_NORMALIZED = 1.5; +const assert_subquadratic = ( + label: string, + inputs: Array, + work_fn: (input: string) => number, +): void => { + const points = inputs.map((input) => ({len: input.length, work: work_fn(input)})); + for (const p of points) assert.isAbove(p.work, 0, `${label}: nonzero work (counter wired)`); + for (let i = 1; i < points.length; i++) { + const size_ratio = points[i]!.len / points[i - 1]!.len; + const work_ratio = points[i]!.work / points[i - 1]!.work; + const normalized = work_ratio / size_ratio; + assert.isBelow( + normalized, + MAX_NORMALIZED, + `${label}: step ${i} normalized=${normalized.toFixed(3)} ` + + `(work ${points[i - 1]!.work}→${points[i]!.work}, len ${points[i - 1]!.len}→${points[i]!.len})`, + ); + } +}; + +describe('parser scan work is sub-quadratic (deterministic)', () => { + test('sync: nested unclosed `[` (Bug 5 revert re-tokenization)', () => { + assert_subquadratic( + 'nested [', + [2000, 4000, 8000].map((n) => '['.repeat(n) + 'text'), + sync_work, + ); + }); + + test('sync: nested same-name `` (Bug 5 revert re-tokenization)', () => { + assert_subquadratic( + 'nested ', + [1000, 2000, 4000].map((n) => ''.repeat(n) + 'X'), + sync_work, + ); + }); + + test('sync: plain prose text scan', () => { + assert_subquadratic( + 'prose', + [2000, 4000, 8000].map((n) => 'the quick brown fox jumps over the lazy dog. '.repeat(n / 45)), + sync_work, + ); + }); + + test('streaming: hold-line code (buffer_index_of memo)', () => { + assert_subquadratic( + 'hold code', + [4000, 8000, 16000].map((chars) => '**a `' + 'x'.repeat(chars)), + (input) => stream_work(input, 64), + ); + }); + + test('streaming: hold-line link (buffer_index_of memo)', () => { + assert_subquadratic( + 'hold link', + [4000, 8000, 16000].map((chars) => '[text](https://example.com/' + 'x'.repeat(chars)), + (input) => stream_work(input, 64), + ); + }); + + test('streaming: mismatched tags (open_tag_counts O(1) bail)', () => { + assert_subquadratic( + 'mismatched tags', + [1000, 2000, 4000].map((n) => + Array.from({length: n}, (_, i) => `x`).join(''), + ), + (input) => stream_work(input, input.length + 1), + ); + }); +}); diff --git a/src/test/mdz_to_svelte.render_parity.test.ts b/src/test/mdz_to_svelte.render_parity.test.ts new file mode 100644 index 0000000..8012636 --- /dev/null +++ b/src/test/mdz_to_svelte.render_parity.test.ts @@ -0,0 +1,88 @@ +/** + * Render parity: `mdz_to_svelte`'s generated markup must match what the + * runtime renderer (`Mdz` → `MdzNodeView`) actually produces. + * + * The render semantics live in three hand-synced implementations + * (`MdzNodeView.svelte`, `MdzStreamNodeView.svelte`, `mdz_to_svelte.ts`); + * the other suites bind trees and preprocessor output, but nothing else + * compares the *markup*. This suite SSRs every fixture through `Mdz` and + * normalizes `mdz_to_svelte`'s Svelte-template output to the equivalent + * HTML, then diffs. + * + * Scope: the plain default configuration (no components/elements, plain + * code/codeblock). Fixtures with unconfigured tags are skipped — the two + * implementations intentionally diverge there (`mdz_to_svelte` emits + * nothing and sets `has_unconfigured_tags`; the runtime renders the + * unregistered-tag fallback). + */ + +import {describe, test, assert} from 'vitest'; +import {render} from 'svelte/server'; +import type {Component} from 'svelte'; + +import MdzComponent from '$lib/Mdz.svelte'; +import {mdz_parse} from '$lib/mdz.ts'; +import {mdz_to_svelte} from '$lib/mdz_to_svelte.ts'; +import {load_fixtures} from './fixtures/mdz/mdz_test_helpers.ts'; + +// narrowed component type — the wrapper's `SvelteHTMLElements` rest-prop +// union is too complex for `render()`'s generic inference +const Mdz = MdzComponent as unknown as Component<{content: string}>; + +// escape_svelte_text's brace escapes — the only source of these forms in +// generated markup (attr expression strings never contain braces: references +// are valid-path-chars only, heading ids are slugs) +const TEXT_OPEN_BRACE = /\{'\{'\}/g; +const TEXT_CLOSE_BRACE = /\{'\}'\}/g; + +const js_unescape = (s: string): string => s.replaceAll(/\\(.)/g, '$1'); +const attr_escape = (s: string): string => s.replaceAll('&', '&').replaceAll('"', '"'); + +/** + * Convert `mdz_to_svelte` output (Svelte template markup, plain default + * config) to the HTML the equivalent runtime template server-renders. + * `resolve()` is the identity here — the test env has no SvelteKit base. + */ +const svelte_markup_to_html = (markup: string): string => + markup + // sentinel the text-level brace escapes so attr regexes can't misfire + .replaceAll(TEXT_OPEN_BRACE, '\u0001') + .replaceAll(TEXT_CLOSE_BRACE, '\u0002') + .replaceAll( + /=\{resolve\('((?:[^'\\]|\\.)*)'\)\}/g, + (_, s: string) => `="${attr_escape(js_unescape(s))}"`, + ) + .replaceAll(/=\{'((?:[^'\\]|\\.)*)'\}/g, (_, s: string) => `="${attr_escape(js_unescape(s))}"`) + .replaceAll(/=\{(\d+)\}/g, '="$1"') + // Svelte SSR prints self-closing void elements without the space + .replaceAll('


    ', '
    ') + .replaceAll('\u0001', '{') + .replaceAll('\u0002', '}'); + +/** SSR body → the wrapper div's inner HTML with hydration comments stripped. */ +const ssr_inner_html = (body: string): string => { + const stripped = body.replaceAll(//gs, ''); + assert.isTrue(stripped.startsWith('
    ') && stripped.endsWith('
    '), 'wrapper div'); + return stripped.slice('
    '.length, -'
    '.length); +}; + +describe('mdz_to_svelte matches the runtime renderer', () => { + test('all fixtures (plain default config)', async () => { + const fixtures = await load_fixtures(); + const failures: Array = []; + let compared = 0; + for (const fixture of fixtures) { + const result = mdz_to_svelte(mdz_parse(fixture.input), {components: {}, elements: {}}); + if (result.has_unconfigured_tags) continue; // intentional divergence — see module docs + const expected = svelte_markup_to_html(result.markup); + const actual = ssr_inner_html(render(Mdz, {props: {content: fixture.input}}).body); + if (expected !== actual) { + failures.push(`${fixture.name}:\n markup→ ${expected}\n ssr→ ${actual}`); + } + compared++; + } + // the corpus is mostly tag-free — a low count means the skip filter broke + assert.isAbove(compared, 300, 'compared fixture count'); + assert.deepEqual(failures.slice(0, 5), [], `${failures.length} of ${compared} diverge`); + }); +}); diff --git a/src/test/mdz_to_svelte.test.ts b/src/test/mdz_to_svelte.test.ts index 2c1c32c..70f2621 100644 --- a/src/test/mdz_to_svelte.test.ts +++ b/src/test/mdz_to_svelte.test.ts @@ -2,7 +2,7 @@ import {test, assert, describe} from 'vitest'; import {escape_svelte_text} from '@fuzdev/fuz_util/svelte_preprocess_helpers.ts'; -import {mdz_parse} from '$lib/mdz.ts'; +import {mdz_parse, type MdzNode} from '$lib/mdz.ts'; import { mdz_to_svelte, type MdzToSvelteOptions, @@ -280,6 +280,29 @@ describe('mdz_to_svelte', () => { const result = convert('[a & b](/path)'); assert.ok(result.markup.includes('>a & b')); }); + + test('unsafe reference renders as plain children, no (defense in depth)', () => { + // `mdz_parse` rejects unsafe references at parse time, so build the node by + // hand — `mdz_to_svelte` is a public entry and must not trust its input, + // matching `MdzNodeView`/`MdzStreamNodeView`. render_parity can't cover this + // (its fixtures are all `mdz_parse` output, which never carries an unsafe ref). + const nodes: Array = [ + { + type: 'Link', + // eslint-disable-next-line no-script-url -- the unsafe protocol under test + reference: 'javascript:alert(1)', + link_type: 'external', + children: [{type: 'Text', content: 'click', start: 0, end: 5}], + start: 0, + end: 20, + }, + ]; + const result = mdz_to_svelte(nodes, {components: {}, elements: {}}); + assert.equal(result.markup, 'click'); + // eslint-disable-next-line no-script-url -- asserting the unsafe protocol is absent + assert.notInclude(result.markup, 'javascript:'); + assert.notInclude(result.markup, ' { diff --git a/src/test/test_helpers.ts b/src/test/test_helpers.ts index ba37491..4e0a374 100644 --- a/src/test/test_helpers.ts +++ b/src/test/test_helpers.ts @@ -1,159 +1,11 @@ /** - * Shared test utilities for vitest tests. - * Provides helpers for component mounting, DOM event creation, test data normalization, - * and generic fixture loading/updating patterns. + * Shared test utilities for vitest tests — generic fixture loading and + * update task patterns. */ -import {mount, unmount, type Component} from 'svelte'; import {readdir, readFile, writeFile} from 'node:fs/promises'; import {join} from 'node:path'; -/** - * Mount a Svelte component for testing. - * Creates a container div, appends it to document.body, and mounts the component. - */ -export const mount_component = >( - Component: Component, - props: TProps, -): {instance: any; container: HTMLElement} => { - const container = document.createElement('div'); - document.body.appendChild(container); - - const instance = mount(Component, { - target: container, - props, - }); - - return {instance, container}; -}; - -/** - * Unmount a component and remove its container from the DOM. - */ -export const unmount_component = async (instance: any, container: HTMLElement): Promise => { - await unmount(instance); - container.remove(); -}; - -/** - * Create a contextmenu (rightclick) mouse event. - */ -export const create_contextmenu_event = ( - x: number, - y: number, - options: MouseEventInit = {}, -): MouseEvent => { - return new MouseEvent('contextmenu', { - bubbles: true, - cancelable: true, - clientX: x, - clientY: y, - ...options, - }); -}; - -/** - * Create a keyboard event. - */ -export const create_keyboard_event = ( - key: string, - options: KeyboardEventInit = {}, -): KeyboardEvent => { - return new KeyboardEvent('keydown', { - bubbles: true, - cancelable: true, - key, - ...options, - }); -}; - -/** - * Create a generic mouse event. - */ -export const create_mouse_event = (type: string, options: MouseEventInit = {}): MouseEvent => { - return new MouseEvent(type, { - bubbles: true, - cancelable: true, - ...options, - }); -}; - -/** - * Set the target property on an event (for testing). - * The target property is readonly, so we need to use Object.defineProperty. - */ -export const set_event_target = (event: Event, target: EventTarget): void => { - Object.defineProperty(event, 'target', { - value: target, - enumerable: true, - configurable: true, - }); -}; - -/** - * Create a touch event with one or more touches. - */ -export const create_touch_event = ( - type: string, - touches: Array<{clientX: number; clientY: number; target?: EventTarget}>, - options: TouchEventInit = {}, -): TouchEvent => { - // Create Touch objects (mocked for testing) - const touch_objects: Array = touches.map((touch, index) => ({ - identifier: index, - clientX: touch.clientX, - clientY: touch.clientY, - screenX: touch.clientX, - screenY: touch.clientY, - pageX: touch.clientX, - pageY: touch.clientY, - radiusX: 0, - radiusY: 0, - rotationAngle: 0, - force: 1, - target: touch.target ?? document.body, - })); - - return new TouchEvent(type, { - bubbles: true, - cancelable: true, - touches: touch_objects, - targetTouches: touch_objects, - changedTouches: touch_objects, - ...options, - }); -}; - -/** - * Normalize an object by removing all undefined values. - * This matches JSON serialization behavior where undefined values are omitted. - * - * Used to compare test results with expected.json files, since JSON.stringify - * automatically removes undefined values when serializing. - * - * Note: Treats both null and undefined as null for comparison consistency. - * - * @param obj - the object to normalize - * @returns the normalized object without undefined values - */ -export const normalize_json = (obj: any): any => { - // Treat both null and undefined as null for comparison consistency - if (obj === null || obj === undefined) return null; - if (Array.isArray(obj)) { - return obj.map(normalize_json); - } - if (typeof obj === 'object') { - const normalized: any = {}; - for (const [key, value] of Object.entries(obj)) { - if (value !== undefined) { - normalized[key] = normalize_json(value); - } - } - return normalized; - } - return obj; -}; - // // Generic Fixture Loading and Update Task Patterns //