Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
## Input

```md
a {b c

x **y** {b c d

# Title {.red

- item {#id
```

## AST

```json
{
"frontmatter": {},
"meta": {},
"nodes": [
[
"p",
{},
"a {b c"
],
[
"p",
{},
"x ",
[
"strong",
{},
"y"
],
" {b c d"
],
[
"h1",
{
"id": "title-red"
},
"Title {.red"
],
[
"ul",
{},
[
"li",
{},
"item {#id"
]
]
]
}
```

## HTML

```html
<p>a {b c</p>
<p>x <strong>y</strong> {b c d</p>
<h1 id="title-red">Title {.red</h1>
<ul>
<li>item {#id</li>
</ul>
```

## Markdown

```md
a \{b c

x **y** \{b c d

# Title \{.red

- item \{#id
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
## Input

```md
---
title: a
--- x
---

Body
```

## AST

```json
{
"frontmatter": {},
"meta": {},
"nodes": [
[
"p",
{},
"Body"
]
]
}
```

## HTML

```html
<p>Body</p>
```

## Markdown

```md
Body
```
39 changes: 39 additions & 0 deletions packages/comark/SPEC/COMARK/frontmatter-invalid-yaml.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
## Input

```md
---
title: [
---

# Hi
```

## AST

```json
{
"frontmatter": {},
"meta": {},
"nodes": [
[
"h1",
{
"id": "hi"
},
"Hi"
]
]
}
```

## HTML

```html
<h1 id="hi">Hi</h1>
```

## Markdown

```md
# Hi
```
12 changes: 11 additions & 1 deletion packages/comark/src/auto-close.ts
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,17 @@ function scanBlocks(src: string, o: Opts): DocState {

const dashes = c0 === 45 /* - */ && en - st === 3 && src.charCodeAt(st + 1) === 45 && src.charCodeAt(st + 2) === 45

if (i === 0 && o.frontmatter && dashes) {
// Matches `parseFrontmatter`, which accepts a leading byte order mark before the opener.
if (
i === 0 &&
o.frontmatter &&
(dashes ||
(c0 === 0xfeff &&
en - st === 4 &&
src.charCodeAt(st + 1) === 45 &&
src.charCodeAt(st + 2) === 45 &&
src.charCodeAt(st + 3) === 45))
) {
fmOpen = true
continue
}
Expand Down
53 changes: 42 additions & 11 deletions packages/comark/src/internal/frontmatter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,27 +4,58 @@ import { parseYaml, stringifyYaml } from './yaml.ts'
const FRONTMATTER_DELIMITER_DEFAULT = '---'
const LF = '\n'
const CR = '\r'
const BOM = '\uFEFF'

/** Extract leading YAML frontmatter (`---`) from markdown content. */
export function parseFrontmatter(content: string) {
let data: Record<string, any> = {}
/**
* Find the line that closes the frontmatter: `---` alone on its line, with optional trailing spaces or tabs.
* @returns The index of the line feed before the delimiter, and the index after the delimiter line's content
*/
function findClosingDelimiter(content: string, from: number) {
let idx = content.indexOf(LF + FRONTMATTER_DELIMITER_DEFAULT, from)
while (idx !== -1) {
let end = idx + 1 + FRONTMATTER_DELIMITER_DEFAULT.length
while (content[end] === ' ' || content[end] === '\t') end++
if (content[end] === CR) end++
if (end === content.length || content[end] === LF) return { idx, end }
idx = content.indexOf(LF + FRONTMATTER_DELIMITER_DEFAULT, idx + 1)
}
}

/** Split leading frontmatter (`---`) from markdown content without parsing its YAML. */
export function splitFrontmatter(content: string): { content: string; frontmatterText: string } {
let frontmatter = ''
if (content.startsWith(FRONTMATTER_DELIMITER_DEFAULT)) {
const idx = content.indexOf(LF + FRONTMATTER_DELIMITER_DEFAULT)
if (idx !== -1) {
const start = content.startsWith(BOM) ? BOM.length : 0
Comment thread
farnabaz marked this conversation as resolved.
Resolved
if (content.startsWith(FRONTMATTER_DELIMITER_DEFAULT, start)) {
const close = findClosingDelimiter(content, start)
if (close) {
const { idx, end } = close
const hasCarriageReturn = content[idx - 1] === CR
frontmatter = content.slice(4, idx - (hasCarriageReturn ? 1 : 0))
frontmatter = content.slice(start + 4, idx - (hasCarriageReturn ? 1 : 0))
if (frontmatter) {
data = parseYaml(frontmatter) ?? {}
content = content.slice(idx + 4 + (hasCarriageReturn ? 1 : 0))
content = content.slice(end)
}
}
}

return { content, frontmatterText: frontmatter }
}

/**
* Extract leading YAML frontmatter (`---`) from markdown content.
* @throws When the frontmatter is not valid YAML
*/
export function parseFrontmatter(content: string): {
content: string
data: Record<string, any>
frontmatterText: string
} {
const split = splitFrontmatter(content)
const data: Record<string, any> = split.frontmatterText ? (parseYaml(split.frontmatterText) ?? {}) : {}

return {
content,
content: split.content,
data,
frontmatterText: frontmatter,
frontmatterText: split.frontmatterText,
}
}

Expand Down
3 changes: 3 additions & 0 deletions packages/comark/src/internal/parse/syntax/props.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ export function searchProps(content: string, index = 0) {
if (content[index] !== '{') throw new Error(`Invalid props, expected \`{\` but got '${content[index]}'`)

const props: [string, string][] = []
let closed = false

// Skip Vue mustache `{{ }}` syntax
if (content[index + 1] === '{') return undefined
Expand All @@ -34,6 +35,7 @@ export function searchProps(content: string, index = 0) {
index += 2
} else if (content[index] === '}') {
index += 1
closed = true
break
} else if (content[index] === ' ') {
index += 1
Expand Down Expand Up @@ -123,5 +125,6 @@ export function searchProps(content: string, index = 0) {
return {
props,
index,
closed,
}
}
10 changes: 10 additions & 0 deletions packages/comark/src/plugins/attributes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ const markdownItInlineProps: PluginSimple = (md) => {
// after text inside it would be rejected too, so it is skipped without
// rescanning the same suffix (keeps O(n)).
const rejectedUntil = new WeakMap<object, number>()
// Start of the first brace group without a closing `}`, per inline state. Its
// scan ran to the end of the source, so a later `{` is text too (keeps O(n)).
const unclosedFrom = new WeakMap<object, number>()
// Set while re-tokenizing a restored brace group: the end of that source is
// not the end of the line, since text followed the group in the original.
let restoring = false
Expand All @@ -46,8 +49,15 @@ const markdownItInlineProps: PluginSimple = (md) => {
lastDelimiter.token === state.tokens.length - 1
if (afterText && !afterCloser && start < (rejectedUntil.get(state) ?? -1)) return false

if (start > (unclosedFrom.get(state) ?? Infinity)) return false

const search = searchProps(state.src, start)
if (!search) return false
// An unclosed group, such as `a {b c`, is text
if (!search.closed) {
unclosedFrom.set(state, Math.min(start, unclosedFrom.get(state) ?? Infinity))
return false
}

const { props, index: end } = search
if (end === start) return false
Expand Down
11 changes: 9 additions & 2 deletions packages/comark/src/plugins/frontmatter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,19 @@
*/

import { defineComarkPlugin } from '../utils/helpers.ts'
import { parseFrontmatter } from '../internal/frontmatter.ts'
import { parseFrontmatter, splitFrontmatter } from '../internal/frontmatter.ts'

export default defineComarkPlugin(() => ({
name: 'frontmatter',
pre(state) {
const { content, data, frontmatterText } = parseFrontmatter(state.markdown)
let result: ReturnType<typeof parseFrontmatter>
try {
result = parseFrontmatter(state.markdown)
} catch (error) {
console.warn(`[comark/plugins/frontmatter] ignoring invalid YAML frontmatter: ${(error as Error).message}`)
result = { ...splitFrontmatter(state.markdown), data: {} }
}
const { content, data, frontmatterText } = result
state.markdown = content
state.frontmatter = data
state.frontmatterText = frontmatterText
Expand Down
5 changes: 5 additions & 0 deletions packages/comark/test/auto-close.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,11 @@ describe('frontmatter', () => {
const expected = '---\ntitle: Test\n---'
expect(autoCloseMarkdown(input, { frontmatter: true })).toBe(expected)
})
it('should complete partial frontmatter after a byte order mark', () => {
const input = '\uFEFF---\ntitle: Test'
const expected = '\uFEFF---\ntitle: Test\n---'
expect(autoCloseMarkdown(input, { frontmatter: true })).toBe(expected)
})
it('should complete a partial closing delimiter (-) when frontmatter option is enabled', () => {
const input = '---\ntitle: Test\n-'
const expected = '---\ntitle: Test\n---'
Expand Down
31 changes: 29 additions & 2 deletions packages/comark/test/frontmatter.test.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import { parseFrontmatter, renderFrontmatter } from '../src/internal/frontmatter'
import { createMarkdownParser, parseMarkdown } from '../src/index'

Expand Down Expand Up @@ -126,7 +126,7 @@ Content`
expect(result.content.trim()).toBe('Content')
})

it('should still throw for malformed YAML in frontmatter', () => {
it('should throw for malformed YAML in frontmatter', () => {
const input = `---
foo: [1,2
---
Expand All @@ -135,6 +135,33 @@ Content`
expect(() => parseFrontmatter(input)).toThrow()
})

it('should warn and strip malformed YAML frontmatter in the frontmatter plugin', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
const input = `---
foo: [1,2
---

Content`
const result = await parseMarkdown(input)
expect(result.frontmatter).toEqual({})
expect(result.nodes).toEqual([['p', {}, 'Content']])
expect(warn).toHaveBeenCalledOnce()
expect(warn.mock.calls[0][0]).toContain('invalid YAML frontmatter')
warn.mockRestore()
})

it('should close frontmatter at a delimiter with trailing spaces', () => {
const result = parseFrontmatter('---\ntitle: Hello\n--- \t\n\nContent')
expect(result.data).toEqual({ title: 'Hello' })
expect(result.content.trim()).toBe('Content')
})

it('should find frontmatter after a byte order mark', () => {
const result = parseFrontmatter('\uFEFF---\ntitle: Hello\n---\n\nContent')
expect(result.data).toEqual({ title: 'Hello' })
expect(result.content.trim()).toBe('Content')
})

it('should handle frontmatter with special characters', () => {
const input = `---
title: "Hello: World"
Expand Down
9 changes: 9 additions & 0 deletions packages/comark/test/streaming.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,15 @@ describe('streaming mode', () => {
expect(result2.frontmatter).toEqual({ title: 'Hello', category: 'Test' })
})

it('closes partial frontmatter after a byte order mark', async () => {
const parse = createMarkdownParser()

const result = await parse('\uFEFF---\ntitle: Hello', { streaming: true })

expect(result.frontmatter).toEqual({ title: 'Hello' })
expect(result.nodes).toEqual([])
})

it('frontmatter is preserved across continuation calls', async () => {
const parse = createMarkdownParser()

Expand Down
Loading
Loading