diff --git a/CHANGELOG.md b/CHANGELOG.md index dc5afbc..2d8f4f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- Indented HTML (most real web pages) now keeps blank lines between blocks; previously + headings, paragraphs, lists and tables after the first ran together with a leading space + (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. + ### Changed - Design docs moved from `specs/` to an OKF v0.2 knowledge bundle in `knowledge/`, validated in CI. diff --git a/crates/markkit/src/html/parser.rs b/crates/markkit/src/html/parser.rs index 6341d29..6cfebd5 100644 --- a/crates/markkit/src/html/parser.rs +++ b/crates/markkit/src/html/parser.rs @@ -51,7 +51,7 @@ impl<'a> HtmlParser<'a> { self.convert_children(content_root, &exclude_selectors, &mut nodes, 0); } - Ok(Document::new(nodes)) + Ok(Document::new(normalize_blocks(nodes))) } fn is_excluded(&self, el: &scraper::ElementRef<'_>, exclude: &[Selector]) -> bool { @@ -181,6 +181,7 @@ impl<'a> HtmlParser<'a> { "blockquote" => { let mut children = Vec::new(); self.convert_children(el, exclude, &mut children, depth); + let children = normalize_blocks(children); if children.is_empty() { return None; } @@ -348,6 +349,7 @@ impl<'a> HtmlParser<'a> { { let mut children = Vec::new(); self.convert_children(el, exclude, &mut children, depth + 1); + let children = normalize_blocks(children); if !children.is_empty() { items.push(ListItem::new(children)); } @@ -461,6 +463,62 @@ fn collapse_whitespace(s: &str) -> String { result } +/// Normalize a block-context node list. +/// +/// Source indentation between block elements arrives as whitespace-only text, +/// which `convert_children` keeps as `Text(" ")` (it matters between inline +/// siblings). Next to a block it is noise that breaks blank-line separation and +/// leaks a leading space into the next line, so: when the list holds any block, +/// drop whitespace-only inline runs and wrap the remaining inline runs in a +/// `Paragraph` (edge whitespace trimmed). Pure-inline lists are returned as is. +fn normalize_blocks(nodes: Vec) -> Vec { + if nodes.iter().all(is_inline) { + return nodes; + } + let mut out = Vec::with_capacity(nodes.len()); + let mut run: Vec = Vec::new(); + for node in nodes { + if is_inline(&node) { + run.push(node); + } else { + flush_inline_run(&mut run, &mut out); + out.push(node); + } + } + flush_inline_run(&mut run, &mut out); + out +} + +fn flush_inline_run(run: &mut Vec, out: &mut Vec) { + let mut children = std::mem::take(run); + trim_edge_text(&mut children); + if !children.is_empty() { + out.push(Node::Paragraph { children }); + } +} + +/// Trim whitespace at the outer edges of an inline run, dropping emptied text nodes. +fn trim_edge_text(children: &mut Vec) { + while let Some(Node::Text(t)) = children.first_mut() { + let trimmed = t.trim_start(); + if trimmed.is_empty() { + children.remove(0); + } else { + *t = trimmed.to_string(); + break; + } + } + while let Some(Node::Text(t)) = children.last_mut() { + let trimmed = t.trim_end(); + if trimmed.is_empty() { + children.pop(); + } else { + *t = trimmed.to_string(); + break; + } + } +} + /// Check if a tag is a transparent container (should be unwrapped in block context). fn is_transparent_container(tag: &str) -> bool { matches!( diff --git a/crates/markkit/tests/fixtures/html/basic_headings.md b/crates/markkit/tests/fixtures/html/basic_headings.md index d4fff1b..3d4c1e4 100644 --- a/crates/markkit/tests/fixtures/html/basic_headings.md +++ b/crates/markkit/tests/fixtures/html/basic_headings.md @@ -1,6 +1,11 @@ # Heading Level 1 - ## Heading Level 2 - ### Heading Level 3 - #### Heading Level 4 - ##### Heading Level 5 - ###### Heading Level 6 + +## Heading Level 2 + +### Heading Level 3 + +#### Heading Level 4 + +##### Heading Level 5 + +###### Heading Level 6 diff --git a/crates/markkit/tests/fixtures/html/blockquotes.md b/crates/markkit/tests/fixtures/html/blockquotes.md index a74c40e..2d1f040 100644 --- a/crates/markkit/tests/fixtures/html/blockquotes.md +++ b/crates/markkit/tests/fixtures/html/blockquotes.md @@ -1,4 +1,7 @@ > A simple blockquote. - > Outer quote -> > Nested quote - > Quote with **bold** and *italic* text. + +> Outer quote +> +> > Nested quote + +> Quote with **bold** and *italic* text. diff --git a/crates/markkit/tests/fixtures/html/code_blocks.md b/crates/markkit/tests/fixtures/html/code_blocks.md index b1b2278..9b5b68d 100644 --- a/crates/markkit/tests/fixtures/html/code_blocks.md +++ b/crates/markkit/tests/fixtures/html/code_blocks.md @@ -2,15 +2,18 @@ plain code block with multiple lines ``` - ```rust + +```rust fn main() { println!("Hello, world!"); } ``` - ```python + +```python def hello(): print("hi") ``` - ``` + +``` preformatted without code tag ``` diff --git a/crates/markkit/tests/fixtures/html/full_article.md b/crates/markkit/tests/fixtures/html/full_article.md index 8c6b1fb..7896b05 100644 --- a/crates/markkit/tests/fixtures/html/full_article.md +++ b/crates/markkit/tests/fixtures/html/full_article.md @@ -1,15 +1,24 @@ # Article Title - This is the **first paragraph** with some [links](/link) and *formatting*. - ## Section One - Section one content with `inline code`. - - Point one + +This is the **first paragraph** with some [links](/link) and *formatting*. + +## Section One + +Section one content with `inline code`. + +- Point one - Point two - ## Section Two - > An important quote. - ```javascript + +## Section Two + +> An important quote. + +```javascript console.log("hello"); ``` - | Col A | Col B | + +| Col A | Col B | | --- | --- | | 1 | 2 | - Final paragraph. + +Final paragraph. diff --git a/crates/markkit/tests/fixtures/html/inline_formatting.md b/crates/markkit/tests/fixtures/html/inline_formatting.md index 12e006d..b0718c6 100644 --- a/crates/markkit/tests/fixtures/html/inline_formatting.md +++ b/crates/markkit/tests/fixtures/html/inline_formatting.md @@ -1,6 +1,11 @@ This is **bold text** in a sentence. - This is *italic text* in a sentence. - This is ~~strikethrough text~~ in a sentence. - This is `inline code` in a sentence. - This has ***bold and italic*** together. - This has **bold**, *italic*, and ~~strikethrough~~. + +This is *italic text* in a sentence. + +This is ~~strikethrough text~~ in a sentence. + +This is `inline code` in a sentence. + +This has ***bold and italic*** together. + +This has **bold**, *italic*, and ~~strikethrough~~. diff --git a/crates/markkit/tests/fixtures/html/links_and_images.md b/crates/markkit/tests/fixtures/html/links_and_images.md index 3f01ed1..84a4cb3 100644 --- a/crates/markkit/tests/fixtures/html/links_and_images.md +++ b/crates/markkit/tests/fixtures/html/links_and_images.md @@ -1,7 +1,13 @@ [Basic Link](https://example.com) - [Link with Title](https://example.com "Example Site") - [Relative Link](/relative) - [**Bold Link Text**](/bold-link) - ![An image](image.jpg) - ![](photo.png) - ![](noalt.gif) + +[Link with Title](https://example.com "Example Site") + +[Relative Link](/relative) + +[**Bold Link Text**](/bold-link) + +![An image](image.jpg) + +![](photo.png) + +![](noalt.gif) diff --git a/crates/markkit/tests/fixtures/html/lists.md b/crates/markkit/tests/fixtures/html/lists.md index 72159b2..d3e67b6 100644 --- a/crates/markkit/tests/fixtures/html/lists.md +++ b/crates/markkit/tests/fixtures/html/lists.md @@ -1,12 +1,15 @@ - Apple - Banana - Cherry - 1. First + +1. First 2. Second 3. Third - - Outer A + +- Outer A - Inner A1 - Inner A2 - Outer B - - **Bold item** with text + +- **Bold item** with text - *Italic item* diff --git a/crates/markkit/tests/fixtures/html/malformed.md b/crates/markkit/tests/fixtures/html/malformed.md index 466deac..69b4a2c 100644 --- a/crates/markkit/tests/fixtures/html/malformed.md +++ b/crates/markkit/tests/fixtures/html/malformed.md @@ -1,7 +1,9 @@ Unclosed paragraph Next paragraph - Mismatched **bold with no close** + +Mismatched **bold with no close** + **Entities: & < > " ' Nested *italic **and bold***** crossed** ** diff --git a/crates/markkit/tests/fixtures/html/tables.md b/crates/markkit/tests/fixtures/html/tables.md index 3aab99a..8756175 100644 --- a/crates/markkit/tests/fixtures/html/tables.md +++ b/crates/markkit/tests/fixtures/html/tables.md @@ -2,13 +2,16 @@ | --- | --- | --- | | Alice | 30 | NYC | | Bob | 25 | LA | - | Feature | Status | + +| Feature | Status | | --- | --- | | **Bold Feature** | *Active* | - | Col1 | Col2 | + +| Col1 | Col2 | | --- | --- | | a | b | - | | | + +| | | | --- | --- | | No | Header | | Row | Data | diff --git a/crates/markkit/tests/html_integration.rs b/crates/markkit/tests/html_integration.rs index a7eee47..afd18f6 100644 --- a/crates/markkit/tests/html_integration.rs +++ b/crates/markkit/tests/html_integration.rs @@ -693,3 +693,49 @@ fn div_with_multiple_blocks() { assert!(md.contains("First")); assert!(md.contains("Second")); } + +// ─── Indented source (inter-block whitespace) ──────────────────────── + +#[test] +fn indented_blocks_keep_blank_line_separation() { + let html = "\n

A

\n

B

\n"; + assert_eq!(convert(html), "# A\n\nB\n"); +} + +#[test] +fn indented_blocks_never_start_with_space() { + let html = + "\n

Title

\n

One

\n

Sub

\n

Two

\n"; + let md = convert(html); + assert_eq!(md, "# Title\n\nOne\n\n## Sub\n\nTwo\n"); + assert!(md.lines().all(|l| !l.starts_with(' ')), "{md:?}"); +} + +#[test] +fn adjacent_lists_separated_by_blank_line() { + let html = "
    \n
  • a
  • \n
\n
    \n
  1. b
  2. \n
"; + assert_eq!(convert(html), "- a\n\n1. b\n"); +} + +#[test] +fn indented_blocks_inside_transparent_containers() { + let html = "
\n
\n

S

\n

P

\n
\n
"; + assert_eq!(convert(html), "## S\n\nP\n"); +} + +#[test] +fn indented_blockquote_with_paragraphs() { + let html = "
\n

One

\n

Two

\n
"; + assert_eq!(convert(html), "> One\n>\n> Two\n"); +} + +#[test] +fn loose_text_between_blocks_becomes_paragraph() { + let html = "
\n

T

\n loose text\n

P

\n
"; + assert_eq!(convert(html), "# T\n\nloose **text**\n\nP\n"); +} + +#[test] +fn inline_only_document_unchanged() { + assert_eq!(convert("Hello world"), "Hello **world**\n"); +} diff --git a/crates/markkit/tests/html_proptest.rs b/crates/markkit/tests/html_proptest.rs index b65c833..1453afa 100644 --- a/crates/markkit/tests/html_proptest.rs +++ b/crates/markkit/tests/html_proptest.rs @@ -11,6 +11,27 @@ fn convert(html: &str) -> String { } proptest! { + /// Source indentation between blocks never changes the output: an indented + /// document renders exactly like its compact form. + #[test] + fn inter_block_whitespace_is_insignificant( + blocks in prop::collection::vec( + (prop::sample::select(vec!["h1", "h2", "p", "blockquote"]), "[a-z]{1,8}"), + 1..8, + ), + indent in "[ \t\n]{1,6}", + ) { + let compact: String = blocks.iter().map(|(t, s)| format!("<{t}>{s}")).collect(); + let indented: String = blocks + .iter() + .map(|(t, s)| format!("{indent}<{t}>{s}")) + .collect::() + + &indent; + let md = convert(&indented); + prop_assert_eq!(&md, &convert(&compact)); + prop_assert!(md.lines().all(|l| !l.starts_with(' ')), "{:?}", md); + } + /// Arbitrary HTML input should never panic. #[test] fn no_panic_on_arbitrary_input(s in "\\PC{0,500}") { diff --git a/knowledge/formats/html-extraction.md b/knowledge/formats/html-extraction.md index 1da7924..c1bcc1a 100644 --- a/knowledge/formats/html-extraction.md +++ b/knowledge/formats/html-extraction.md @@ -107,6 +107,14 @@ The `collapse_whitespace` function normalizes text nodes: 3. Trailing whitespace: preserved as single space if original had trailing whitespace AND result is non-empty 4. Whitespace-only text nodes: if adjacent to existing content, emit single space; otherwise elided +Block-context normalization (`normalize_blocks`, applied to the document body, +blockquote children and list-item children): when a node list contains any block, +whitespace-only inline runs between blocks are dropped and remaining inline runs +are wrapped in a `Paragraph` with edge whitespace trimmed. This is what keeps +indented source (`\n

A

\n

B

`) rendering as `# A\n\nB` +instead of `# A\n B`. Pure-inline lists are left untouched, so whitespace between +inline siblings survives. + Trimming at render time: - `trim_trailing_inline`: strips leading/trailing spaces from inline content within headings and paragraphs - `normalize_output`: collapses multiple blank lines to one, trims document-level leading/trailing blanks @@ -155,6 +163,9 @@ providing defense-in-depth. - Unclosed/mismatched tags: html5ever auto-repairs per HTML5 spec - Empty containers: all block-level containers with no content are elided - HTML entities: decoded by html5ever before reaching parser +- Blocks inside inline formatting: after an unclosed ``, html5ever's adoption + agency wraps following paragraphs in the formatting element; they render inside the + emphasis markers (known limitation, see `malformed` fixture) ## See also diff --git a/knowledge/foundations/ir.md b/knowledge/foundations/ir.md index a0eaeab..f861201 100644 --- a/knowledge/foundations/ir.md +++ b/knowledge/foundations/ir.md @@ -82,7 +82,11 @@ Input format -> `Document`. Each parser (HTML, future PDF/DOCX) produces the sam ### 2. Normalize -Implicit during rendering. The `normalize_output` function: +In the parser, block-context node lists go through `normalize_blocks`: no +whitespace-only inline nodes between blocks, and loose inline runs next to blocks +become `Paragraph`s (see [HTML Extraction](../formats/html-extraction.md)). + +At render time, the `normalize_output` function: - Collapses multiple consecutive blank lines to one - Trims leading/trailing blank lines @@ -105,6 +109,7 @@ Implicit during rendering. The `normalize_output` function: |-----------|-------------| | Heading level is 1-6 | Parser clamps via `tag[1..].parse::()` | | No empty Text nodes after whitespace collapse | `collapse_whitespace` returns empty string for whitespace-only; parser skips empty | +| Block lists hold no bare inline runs | `normalize_blocks` drops whitespace-only runs, wraps others in `Paragraph` | | Empty containers are elided | Parser returns `None` for headings, paragraphs, lists, etc. with no children | | Depth bounded at 200 | `MAX_DEPTH` constant; at limit, subtree flattened to text | | No executable content in IR | Script, style, iframe, etc. stripped by parser | diff --git a/knowledge/log.md b/knowledge/log.md index 1afc07f..8701f9f 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). * **OKF migration**: `specs/` became this OKF v0.2 bundle, grouped into `foundations/`, `formats/`, `security/`, `runtimes/`, `operations/`. Every concept gained frontmatter and a See also section; `just check-okf` and CI enforce conformance. See [Knowledge Maintenance Contract](knowledge-contract.md). * **Accuracy pass**: [Python Package](runtimes/python-package.md) now names the shipped `markkit` module; [Examples](operations/examples.md) lists the four real examples; [Evaluation Harness](operations/eval.md) is marked as a stub; [Threat Model](security/threat-model.md) T3 records that `http_client` is a default feature and SSRF mitigations are still open. * **Maintenance contract**: [Maintenance](operations/maintenance.md) adopts the analyze, fix, ship invocation contract and a workflow credential audit. diff --git a/knowledge/operations/testing.md b/knowledge/operations/testing.md index 77e5e90..e639d7f 100644 --- a/knowledge/operations/testing.md +++ b/knowledge/operations/testing.md @@ -33,7 +33,8 @@ each security test states exactly which bytes it feeds the parser. - **Unit**: detection, element conversion, builder, error paths - **Integration**: HTML and office pipelines end to end, CLI - **Property** (`proptest`): arbitrary input never panics, output never contains - `