diff --git a/packages/comark/src/internal/shiki.ts b/packages/comark/src/internal/shiki.ts index 05e55bbd..dd0d420d 100644 --- a/packages/comark/src/internal/shiki.ts +++ b/packages/comark/src/internal/shiki.ts @@ -276,6 +276,11 @@ function tokensToSpans(line: ThemedToken[]): Node[] { return spans } +// The streaming parser hands the same raw node to every update while its source +// is unchanged, so a plugin keeps its tokens by raw node. Output nodes are still +// rebuilt every time, since callers and later plugins may edit them in place. +type TokensCache = WeakMap }> + /** * Apply syntax highlighting to all code blocks in a Comark tree * Uses codeToTokens API with batched async operations @@ -284,7 +289,8 @@ export async function highlightCodeBlocks( tree: MarkdownDocument, options: ShikiOptions = {}, defaultThemeLoaders: ShikiThemeLoader[] = [], - defaultLanguageLoaders: ShikiLanguageLoader[] = [] + defaultLanguageLoaders: ShikiLanguageLoader[] = [], + tokensCache?: TokensCache ): Promise { interface CodeBlockRef { node: Node @@ -396,10 +402,15 @@ export async function highlightCodeBlocks( codeChildren = (preNode[2] as ElementNode).slice(2) as Node[] } else { // Fast path: build Nodes directly from tokens, skipping hast - const result = codeToTokens(hl, code, { - lang: language, - themes: themeOptions, - }) + const cached = tokensCache?.get(node as ElementNode) + const result = + cached?.code === code && cached.language === language + ? cached.result + : codeToTokens(hl, code, { + lang: language, + themes: themeOptions, + }) + tokensCache?.set(node as ElementNode, { code, language, result }) classStr = `shiki ${result.themeName || ''}` const tokenLines = result.tokens @@ -544,15 +555,19 @@ export function createShikiPlugin( defaultThemeLoaders: ShikiThemeLoader[] = [], defaultLanguageLoaders: ShikiLanguageLoader[] = [] ) { - return defineComarkPlugin((options) => ({ - name: 'shiki', - async post(state) { - state.tree = await highlightCodeBlocks( - state.tree, - (options || {}) as ShikiOptions, - defaultThemeLoaders, - defaultLanguageLoaders - ) - }, - })) + return defineComarkPlugin((options) => { + const tokensCache: TokensCache = new WeakMap() + return { + name: 'shiki', + async post(state) { + state.tree = await highlightCodeBlocks( + state.tree, + (options || {}) as ShikiOptions, + defaultThemeLoaders, + defaultLanguageLoaders, + tokensCache + ) + }, + } + }) } diff --git a/packages/comark/test/plugins/shiki-streaming.test.ts b/packages/comark/test/plugins/shiki-streaming.test.ts new file mode 100644 index 00000000..3a406132 --- /dev/null +++ b/packages/comark/test/plugins/shiki-streaming.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it, vi } from 'vitest' +import { codeToTokens } from 'shiki/core' +import { createMarkdownParser } from '../../src/index' +import highlight from '../../src/plugins/highlight' +import type { ElementNode } from '../../src/types' + +vi.mock('shiki/core', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, codeToTokens: vi.fn(actual.codeToTokens) } +}) + +describe('shiki while streaming', () => { + it('tokenizes an unchanged fence once while later content streams, without sharing output nodes', async () => { + const parse = createMarkdownParser({ plugins: [highlight()] }) + const source = (tail: string) => `\`\`\`ts\nconst a = 1\n\`\`\`\n\n${tail}` + const before = vi.mocked(codeToTokens).mock.calls.length + const first = await parse(source('a'), { streaming: true }) + expect(vi.mocked(codeToTokens).mock.calls.length).toBe(before + 1) + // A later plugin or the caller editing the output must not leak into the next update. + ;(first.nodes[0] as ElementNode)[1].class = 'edited' + const second = await parse(source('ab'), { streaming: true }) + + expect(vi.mocked(codeToTokens).mock.calls.length).toBe(before + 1) + expect((second.nodes[0] as ElementNode)[1].class).toMatch(/^shiki /) + expect(second.nodes[0]).not.toBe(first.nodes[0]) + }) +})