Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions packages/comark/src/internal/shiki.ts
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,11 @@ function tokensToSpans(line: ThemedToken[]): Node[] {
return spans
}

// The streaming parser hands the same raw node to every update while its source
// is unchanged, so a plugin keeps its tokens by raw node. Output nodes are still
// rebuilt every time, since callers and later plugins may edit them in place.
type TokensCache = WeakMap<ElementNode, { code: string; language: string; result: ReturnType<typeof codeToTokens> }>

/**
* Apply syntax highlighting to all code blocks in a Comark tree
* Uses codeToTokens API with batched async operations
Expand All @@ -284,7 +289,8 @@ export async function highlightCodeBlocks(
tree: MarkdownDocument,
options: ShikiOptions = {},
defaultThemeLoaders: ShikiThemeLoader[] = [],
defaultLanguageLoaders: ShikiLanguageLoader[] = []
defaultLanguageLoaders: ShikiLanguageLoader[] = [],
tokensCache?: TokensCache
): Promise<MarkdownDocument> {
interface CodeBlockRef {
node: Node
Expand Down Expand Up @@ -396,10 +402,15 @@ export async function highlightCodeBlocks(
codeChildren = (preNode[2] as ElementNode).slice(2) as Node[]
} else {
// Fast path: build Nodes directly from tokens, skipping hast
const result = codeToTokens(hl, code, {
lang: language,
themes: themeOptions,
})
const cached = tokensCache?.get(node as ElementNode)
const result =
cached?.code === code && cached.language === language
? cached.result
: codeToTokens(hl, code, {
lang: language,
themes: themeOptions,
})
tokensCache?.set(node as ElementNode, { code, language, result })
classStr = `shiki ${result.themeName || ''}`

const tokenLines = result.tokens
Expand Down Expand Up @@ -544,15 +555,19 @@ export function createShikiPlugin<TOptions extends ShikiOptions = ShikiOptions>(
defaultThemeLoaders: ShikiThemeLoader[] = [],
defaultLanguageLoaders: ShikiLanguageLoader[] = []
) {
return defineComarkPlugin<TOptions>((options) => ({
name: 'shiki',
async post(state) {
state.tree = await highlightCodeBlocks(
state.tree,
(options || {}) as ShikiOptions,
defaultThemeLoaders,
defaultLanguageLoaders
)
},
}))
return defineComarkPlugin<TOptions>((options) => {
const tokensCache: TokensCache = new WeakMap()
return {
name: 'shiki',
async post(state) {
state.tree = await highlightCodeBlocks(
state.tree,
(options || {}) as ShikiOptions,
defaultThemeLoaders,
defaultLanguageLoaders,
tokensCache
)
},
}
})
}
27 changes: 27 additions & 0 deletions packages/comark/test/plugins/shiki-streaming.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { describe, expect, it, vi } from 'vitest'
import { codeToTokens } from 'shiki/core'
import { createMarkdownParser } from '../../src/index'
import highlight from '../../src/plugins/highlight'
import type { ElementNode } from '../../src/types'

vi.mock('shiki/core', async (importOriginal) => {
const actual = await importOriginal<typeof import('shiki/core')>()
return { ...actual, codeToTokens: vi.fn(actual.codeToTokens) }
})

describe('shiki while streaming', () => {
it('tokenizes an unchanged fence once while later content streams, without sharing output nodes', async () => {
const parse = createMarkdownParser({ plugins: [highlight()] })
const source = (tail: string) => `\`\`\`ts\nconst a = 1\n\`\`\`\n\n${tail}`
const before = vi.mocked(codeToTokens).mock.calls.length
const first = await parse(source('a'), { streaming: true })
expect(vi.mocked(codeToTokens).mock.calls.length).toBe(before + 1)
// A later plugin or the caller editing the output must not leak into the next update.
;(first.nodes[0] as ElementNode)[1].class = 'edited'
const second = await parse(source('ab'), { streaming: true })

expect(vi.mocked(codeToTokens).mock.calls.length).toBe(before + 1)
expect((second.nodes[0] as ElementNode)[1].class).toMatch(/^shiki /)
expect(second.nodes[0]).not.toBe(first.nodes[0])
})
})
Loading