From 32f53d355aebcaec413fde746eb01bbfb46243cd Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:53:56 +0000 Subject: [PATCH 1/4] fix(spreadsheet): single-row sheets, workbook titles, XLSB detection - Single-row sheets render the row as data under Excel column letters - Extraction.title from XLSX core.xml, ODS meta.xml, XLS SummaryInformation - XLSB content type / extension / container detected as Format::Xlsb, not Xlsx - XLSX without _rels/.rels gets the standard package relationship - Release profile uses panic = "unwind" so catch_unwind guards work - Document the trailing newline as intended Part of #11 --- CHANGELOG.md | 9 + Cargo.toml | 5 +- crates/markkit/src/extract.rs | 14 +- crates/markkit/src/format.rs | 21 ++ crates/markkit/src/lib.rs | 3 +- crates/markkit/src/spreadsheet.rs | 273 +++++++++++++++++- crates/markkit/tests/common/mod.rs | 61 +++- crates/markkit/tests/document_security.rs | 20 +- .../markkit/tests/spreadsheet_integration.rs | 118 +++++++- knowledge/formats/format-support.md | 29 +- knowledge/log.md | 2 + knowledge/security/threat-model.md | 8 +- 12 files changed, 532 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b78190..fc29176 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- Spreadsheets: a sheet with a single row now renders that row as data under Excel column letters + (`| A | B |`) instead of as a header with no rows. +- Spreadsheets: `Extraction.title` is filled from document properties (XLSX `docProps/core.xml`, + ODS `meta.xml`, XLS `SummaryInformation`). +- `Format::from_content_type` detects XLSB (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) + as the new `Format::Xlsb` instead of `Xlsx`; `.xlsb` files and `xl/workbook.bin` containers too. +- XLSX packages missing `_rels/.rels` no longer fail inside calamine. +- Release builds use `panic = "unwind"`, so a parser panic becomes an error instead of aborting the + CLI or the host Python/Node process. - Indented HTML (most real web pages) now keeps blank lines between blocks; previously headings, paragraphs, lists and tables after the first ran together with a leading space (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. diff --git a/Cargo.toml b/Cargo.toml index a67f998..b16e1da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,5 +68,8 @@ proptest = "1" [profile.release] lto = "thin" codegen-units = 1 -panic = "abort" +# Unwind (not abort): `extract_document` wraps third-party parsers in +# `catch_unwind`, and PyO3/NAPI turn panics into exceptions. Abort would let one +# malformed document kill the CLI or the host Python/Node process. +panic = "unwind" strip = true diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 06ce32b..b627752 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -8,9 +8,11 @@ use crate::format::{DocumentHint, Format}; #[derive(Debug, Clone, PartialEq, Serialize)] #[non_exhaustive] pub struct Extraction { - /// Rendered Markdown. + /// Rendered Markdown. Non-empty output always ends with exactly one `\n` + /// (POSIX text convention, safe to concatenate); empty documents render `""`. pub markdown: String, - /// Document title when known (DOCX core properties or `Title` style). + /// Document title when known: DOCX core properties or `Title` style, + /// XLSX `docProps/core.xml`, ODS `meta.xml`, XLS `SummaryInformation`. pub title: Option, /// Format the document was parsed as. pub format: Format, @@ -50,7 +52,8 @@ impl Extraction { } /// Run a parser, converting a panic into an error (defence in depth for -/// third-party parsers; only effective when built with `panic = "unwind"`). +/// third-party parsers; only effective when built with `panic = "unwind"`, which +/// markkit's own release profile uses). #[cfg(any(feature = "docx", feature = "spreadsheet"))] fn guarded( what: &str, @@ -181,7 +184,9 @@ impl Markkit { } Format::Pdf => self.extract_pdf_sync(bytes), Format::Docx => self.extract_docx(bytes), - Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), + Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => { + self.extract_spreadsheet(bytes) + } Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -222,6 +227,7 @@ impl Markkit { })?; let mut renderer = crate::render::MarkdownRenderer::new(); let mut e = Extraction::new(renderer.render(&out.doc), out.format); + e.title = out.title; e.warnings = out.warnings; e.stats.sheets = Some(out.sheets); e.stats.rows = Some(out.rows); diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index cf632a7..509604c 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -22,6 +22,8 @@ pub enum Format { Doc, /// Office Open XML spreadsheet (`.xlsx`, `.xlsm`, `.xltx`). Xlsx, + /// Excel binary workbook (`.xlsb`). + Xlsb, /// Legacy Excel 97-2003 binary (`.xls`). Xls, /// OpenDocument spreadsheet (`.ods`). @@ -76,6 +78,7 @@ impl Format { Format::Docx => "docx", Format::Doc => "doc", Format::Xlsx => "xlsx", + Format::Xlsb => "xlsb", Format::Xls => "xls", Format::Ods => "ods", Format::Pptx => "pptx", @@ -98,6 +101,8 @@ impl Format { Format::Docx } else if ct.contains("msword") { Format::Doc + } else if ct.contains("ms-excel.sheet.binary") { + Format::Xlsb } else if ct.contains("spreadsheetml") || ct.contains("ms-excel.sheet") { Format::Xlsx } else if ct.contains("ms-excel") { @@ -125,6 +130,7 @@ impl Format { "docx" | "docm" | "dotx" | "dotm" => Format::Docx, "doc" | "dot" => Format::Doc, "xlsx" | "xlsm" | "xltx" | "xltm" => Format::Xlsx, + "xlsb" => Format::Xlsb, "xls" | "xlt" => Format::Xls, "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, @@ -198,6 +204,8 @@ fn sniff_zip(bytes: &[u8]) -> Format { Format::Ods } else if contains(bytes, b"word/document") { Format::Docx + } else if contains(bytes, b"xl/workbook.bin") { + Format::Xlsb } else if contains(bytes, b"xl/workbook.") { Format::Xlsx } else if contains(bytes, b"ppt/presentation.") { @@ -245,6 +253,18 @@ mod tests { ); assert_eq!(Format::from_content_type("application/pdf"), Format::Pdf); assert_eq!(Format::from_content_type("text/plain"), Format::PlainText); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.binary.macroEnabled.12"), + Format::Xlsb + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.macroEnabled.12"), + Format::Xlsx + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel"), + Format::Xls + ); } #[test] @@ -252,6 +272,7 @@ mod tests { assert_eq!(Format::from_extension("html"), Format::Html); assert_eq!(Format::from_extension("PDF"), Format::Pdf); assert_eq!(Format::from_extension("docx"), Format::Docx); + assert_eq!(Format::from_extension("XLSB"), Format::Xlsb); } #[test] diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index d6453e2..1ed8349 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -56,7 +56,8 @@ pub mod ir; pub mod render; #[cfg(feature = "spreadsheet")] mod spreadsheet; -#[cfg(feature = "docx")] +#[cfg(any(feature = "docx", feature = "spreadsheet"))] +#[cfg_attr(not(feature = "docx"), allow(dead_code))] // spreadsheet uses only `find`/`text` mod xml; pub use config::MarkkitBuilder; diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index 2ca280c..9afba58 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -34,6 +34,7 @@ use crate::render::escape_table_cell; pub(crate) struct SheetOutput { pub doc: Document, pub format: Format, + pub title: Option, pub warnings: Vec, pub sheets: usize, pub rows: usize, @@ -63,7 +64,8 @@ enum Container { } pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result { - let container = if bytes.starts_with(b"PK") { + let mut patched = None; + let (container, title) = if bytes.starts_with(b"PK") { let mut zip = ZipGuard::open(bytes, config).map_err(archive_err)?; let has = |zip: &ZipGuard<'_>, name: &str| zip.names().any(|n| n.eq_ignore_ascii_case(name)); @@ -79,20 +81,29 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result Result out.warn(format!("sheet_error: sheet '{name}': {e}")), } } - out.finish(Format::Xlsx) + out.finish(Format::Xlsx, title) } Container::Ods => { let mut wb: Ods<_> = Ods::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Ods) + out.finish(Format::Ods, title) } Container::Xls => { let mut wb: Xls<_> = Xls::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Xls) + out.finish(Format::Xls, title) } } } @@ -217,6 +228,19 @@ fn fmt_ref(d: &DataRef<'_>) -> String { // ─── Grid collection (bounded) ────────────────────────────────────── +/// Excel column letters for a 0-based index (`0` → `A`, `26` → `AA`). +fn col_name(c: u32) -> String { + let mut n = u64::from(c) + 1; + let mut out = Vec::new(); + while n > 0 { + n -= 1; + out.push(b'A' + (n % 26) as u8); + n /= 26; + } + out.reverse(); + String::from_utf8(out).unwrap_or_default() +} + struct SheetTable { header: Vec, rows: Vec>, @@ -298,10 +322,19 @@ impl Grid { } row }); - let header = rows.next()?; + let first = rows.next()?; + let rest: Vec> = rows.collect(); + // GFM tables need a header row. A sheet with a single row is data, not a + // header: label its columns A, B, … so the row renders as a data row. + let (header, rows) = if rest.is_empty() { + let letters = (min_col..min_col + width as u32).map(col_name).collect(); + (letters, vec![first]) + } else { + (first, rest) + }; Some(SheetTable { header, - rows: rows.collect(), + rows, truncated_rows: self.truncated_rows, truncated_cols: self.truncated_cols, }) @@ -395,10 +428,11 @@ impl<'c> Collector<'c> { }); } - fn finish(self, format: Format) -> Result { + fn finish(self, format: Format, title: Option) -> Result { Ok(SheetOutput { doc: Document::new(self.nodes), format, + title, warnings: self.warnings, sheets: self.sheets, rows: self.rows, @@ -406,6 +440,128 @@ impl<'c> Collector<'c> { } } +// ─── Package fixes and document title ─────────────────────────────── + +/// Longest title kept (characters); longer titles are cut. +const MAX_TITLE_CHARS: usize = 1024; + +/// calamine locates the workbook through `_rels/.rels` and fails without it. +/// Rebuild the (already validated) archive with the standard relationship to +/// `xl/workbook.xml`; entries are copied raw, without recompression. +fn with_package_rels(zip: &mut ZipGuard<'_>) -> Result, Error> { + const RELS: &str = r#""#; + let zerr = |e: zip::result::ZipError| err(format!("cannot rebuild XLSX package: {e}")); + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + for i in 0..zip.archive.len() { + let entry = zip.archive.by_index_raw(i).map_err(zerr)?; + w.raw_copy_file(entry).map_err(zerr)?; + } + let opts = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + w.start_file("_rels/.rels", opts).map_err(zerr)?; + std::io::Write::write_all(&mut w, RELS.as_bytes()) + .map_err(|e| err(format!("cannot rebuild XLSX package: {e}")))?; + Ok(w.finish().map_err(zerr)?.into_inner()) +} + +fn clean_title(raw: &str) -> Option { + let t: String = raw + .split_whitespace() + .collect::>() + .join(" ") + .chars() + .take(MAX_TITLE_CHARS) + .collect(); + (!t.is_empty()).then_some(t) +} + +/// `dc:title` from OOXML `docProps/core.xml` or ODF `meta.xml`. Best effort: +/// a missing or malformed part yields `None`; only archive limits are errors. +fn read_dc_title(zip: &mut ZipGuard<'_>, part: &str) -> Result, Error> { + let Some(xml) = zip.read(part).map_err(archive_err)? else { + return Ok(None); + }; + Ok(dc_title(&xml)) +} + +fn dc_title(xml: &[u8]) -> Option { + let root = crate::xml::parse(xml).ok()?; + clean_title(&root.find("title")?.text()) +} + +/// Title (PIDSI_TITLE) from the XLS `\u{5}SummaryInformation` property set +/// ([MS-OLEPS]). Best effort: anything malformed yields `None`. +fn xls_title(comp: &mut cfb::CompoundFile) -> Option { + const CAP: u64 = 1024 * 1024; + let mut data = Vec::new(); + comp.open_stream("/\u{5}SummaryInformation") + .ok()? + .take(CAP) + .read_to_end(&mut data) + .ok()?; + summary_title(&data) +} + +fn summary_title(d: &[u8]) -> Option { + const PID_CODEPAGE: u32 = 1; + const PID_TITLE: u32 = 2; + const VT_I2: u32 = 0x0002; + const VT_LPSTR: u32 = 0x001E; + const VT_LPWSTR: u32 = 0x001F; + if read_u16(d, 0)? != 0xFFFE || read_u32(d, 24)? == 0 { + return None; + } + let set = read_u32(d, 44)? as usize; + let count = read_u32(d, set.checked_add(4)?)? as usize; + let mut codepage = None; + let mut title_at = None; + for i in 0..count.min(1024) { + let entry = set.checked_add(8 + i * 8)?; + let (pid, off) = (read_u32(d, entry)?, read_u32(d, entry + 4)? as usize); + let at = set.checked_add(off)?; + match pid { + PID_CODEPAGE if read_u32(d, at)? & 0xFFFF == VT_I2 => { + codepage = Some(read_u16(d, at + 4)?); + } + PID_TITLE => title_at = Some(at), + _ => {} + } + } + let at = title_at?; + let typ = read_u32(d, at)? & 0xFFFF; + let n = read_u32(d, at + 4)? as usize; + let start = at + 8; + let raw = match typ { + VT_LPWSTR => { + let bytes = d.get(start..start.checked_add(n.checked_mul(2)?)?)?; + let units: Vec = bytes + .chunks_exact(2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .collect(); + String::from_utf16_lossy(&units) + } + VT_LPSTR => { + let bytes = d.get(start..start.checked_add(n)?)?; + match codepage { + Some(1200) => { + let units: Vec = bytes + .chunks_exact(2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .collect(); + String::from_utf16_lossy(&units) + } + // UTF-8 when valid, otherwise Latin-1 (close to Windows-1252). + _ => match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), + }, + } + } + _ => return None, + }; + clean_title(raw.trim_end_matches('\0')) +} + // ─── XLS pre-scan (BIFF records) ──────────────────────────────────── fn read_u16(d: &[u8], at: usize) -> Option { @@ -429,9 +585,10 @@ fn records(stream: &[u8]) -> impl Iterator { }) } -fn prescan_xls(bytes: &[u8], config: &MarkkitBuilder) -> Result<(), Error> { - let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)) - .map_err(|e| err(format!("invalid XLS container: {e}")))?; +fn prescan_xls( + comp: &mut cfb::CompoundFile, + config: &MarkkitBuilder, +) -> Result<(), Error> { let name = ["/Workbook", "/Book", "/WORKBOOK", "/BOOK"] .into_iter() .find(|n| comp.is_stream(n)) @@ -705,6 +862,92 @@ mod tests { assert!(t.truncated_rows); } + #[test] + fn column_names() { + assert_eq!(col_name(0), "A"); + assert_eq!(col_name(25), "Z"); + assert_eq!(col_name(26), "AA"); + assert_eq!(col_name(16_383), "XFD"); + assert_eq!(col_name(701), "ZZ"); + assert_eq!(col_name(702), "AAA"); + assert!(!col_name(u32::MAX).is_empty()); + } + + #[test] + fn single_row_becomes_data_under_column_letters() { + let cfg = MarkkitBuilder::default(); + let mut g = Grid::new(&cfg); + g.add(4, 2, "x".into()); + g.add(4, 3, "y".into()); + let t = g.into_table().unwrap(); + assert_eq!(t.header, vec!["C", "D"]); + assert_eq!(t.rows, vec![vec!["x".to_string(), "y".to_string()]]); + } + + #[test] + fn dc_title_reads_and_normalizes() { + let core = br#" Q3 + Budget & Plan "#; + assert_eq!(dc_title(core).as_deref(), Some("Q3 Budget & Plan")); + assert_eq!(dc_title(b" "), None); + assert_eq!(dc_title(b""), None); + assert_eq!(dc_title(b"\xff\xfe<<"), None); + let long = format!("{}", "x".repeat(5000)); + assert_eq!(dc_title(long.as_bytes()).unwrap().len(), MAX_TITLE_CHARS); + } + + fn summary_info(codepage: u16, typ: u32, value: &[u8], count: u32) -> Vec { + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + // Property set: size, count, 2 × (pid, offset), codepage, title. + let mut set = Vec::new(); + set.extend_from_slice(&0u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&1u32.to_le_bytes()); + set.extend_from_slice(&24u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&32u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&u32::from(codepage).to_le_bytes()); + set.extend_from_slice(&typ.to_le_bytes()); + set.extend_from_slice(&count.to_le_bytes()); + set.extend_from_slice(value); + d.extend_from_slice(&set); + d + } + + #[test] + fn summary_title_decodes_codepages() { + let d = summary_info(1252, 0x1E, b"Sales\0", 6); + assert_eq!(summary_title(&d).as_deref(), Some("Sales")); + let d = summary_info(1252, 0x1E, b"Caf\xe9\0", 5); + assert_eq!(summary_title(&d).as_deref(), Some("Café")); + let d = summary_info(65001, 0x1E, "Звіт\0".as_bytes(), 9); + assert_eq!(summary_title(&d).as_deref(), Some("Звіт")); + let wide: Vec = "Hi\0".encode_utf16().flat_map(u16::to_le_bytes).collect(); + let d = summary_info(1200, 0x1F, &wide, 3); + assert_eq!(summary_title(&d).as_deref(), Some("Hi")); + } + + #[test] + fn summary_title_rejects_malformed() { + assert_eq!(summary_title(b""), None); + assert_eq!(summary_title(&[0xFE, 0xFF, 0, 0]), None); + // Length pointing past the stream. + let d = summary_info(1252, 0x1E, b"abc", u32::MAX); + assert_eq!(summary_title(&d), None); + // Property offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[48 + 20..48 + 24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + // Section offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[44..48].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + } + #[test] fn biff_scan_rejects_inverted_mulrk() { let mut s = Vec::new(); diff --git a/crates/markkit/tests/common/mod.rs b/crates/markkit/tests/common/mod.rs index 47bbd2b..268d897 100644 --- a/crates/markkit/tests/common/mod.rs +++ b/crates/markkit/tests/common/mod.rs @@ -416,15 +416,70 @@ pub fn biff8_workbook(sheets: &[XlsSheet]) -> Vec { /// Wrap a workbook stream into an OLE2 compound file. pub fn xls_from_stream(stream: &[u8]) -> Vec { + ole2(&[("/Workbook", stream)]) +} + +/// Build an OLE2 compound file from `(path, data)` streams. +pub fn ole2(streams: &[(&str, &[u8])]) -> Vec { let mut comp = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); - { - let mut s = comp.create_stream("/Workbook").unwrap(); - s.write_all(stream).unwrap(); + for (path, data) in streams { + let mut s = comp.create_stream(path).unwrap(); + s.write_all(data).unwrap(); } comp.flush().unwrap(); comp.into_inner().into_inner() } +/// `\u{5}SummaryInformation` property set with a code page and a VT_LPSTR title. +pub fn summary_information(codepage: u16, title: &[u8]) -> Vec { + let mut value = title.to_vec(); + value.push(0); + let mut set = Vec::new(); + for v in [ + 0u32, + 2, + 1, + 24, + 2, + 32, + 2, + u32::from(codepage), + 0x1E, + value.len() as u32, + ] { + set.extend_from_slice(&v.to_le_bytes()); + } + set.extend_from_slice(&value); + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + d.extend_from_slice(&set); + d +} + +/// Rebuild a ZIP without the `drop` entries and with `add` appended. +pub fn rezip(bytes: &[u8], drop: &[&str], add: &[(&str, &[u8])]) -> Vec { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + let mut entries = Vec::new(); + for i in 0..zip.len() { + let mut f = zip.by_index(i).unwrap(); + let name = f.name().unwrap().to_string(); + if drop.contains(&name.as_str()) { + continue; + } + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut f, &mut data).unwrap(); + entries.push((name, data)); + } + let mut refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(n, d)| (n.as_str(), d.as_slice())) + .collect(); + refs.extend_from_slice(add); + zip_bytes(&refs) +} + pub fn xls(sheets: &[XlsSheet]) -> Vec { xls_from_stream(&biff8_workbook(sheets)) } diff --git a/crates/markkit/tests/document_security.rs b/crates/markkit/tests/document_security.rs index 693ea25..ce64334 100644 --- a/crates/markkit/tests/document_security.rs +++ b/crates/markkit/tests/document_security.rs @@ -319,6 +319,24 @@ fn valid_fixtures() -> Vec> { dims: Some((0, 1, 0, 2)), cells: vec![XlsCell::Label(0, 0, "a"), XlsCell::Num(0, 1, 2.0)], }])); + // Title sources: XLS SummaryInformation, XLSX core.xml + no `_rels/.rels`. + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "a")], + }]); + let summary = summary_information(1252, b"Title"); + v.push(ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ])); + let core = b"T"; + let book = xlsx_with( + &[Sheet::new("S", vec![vec![Cell::S("a")]])], + &[], + &[("docProps/core.xml", core)], + ); + v.push(rezip(&book, &["_rels/.rels"], &[])); } v } @@ -379,7 +397,7 @@ proptest! { /// Byte flips in valid documents never panic. #[test] fn mutated_fixtures_never_panic( - idx in 0usize..4, + idx in 0usize..6, flips in proptest::collection::vec((any::(), any::()), 1..16), ) { let fixtures = valid_fixtures(); diff --git a/crates/markkit/tests/spreadsheet_integration.rs b/crates/markkit/tests/spreadsheet_integration.rs index d976897..111e8c7 100644 --- a/crates/markkit/tests/spreadsheet_integration.rs +++ b/crates/markkit/tests/spreadsheet_integration.rs @@ -50,9 +50,10 @@ fn xlsx_multiple_sheets_and_empty_sheet_skipped() { let e = extract(&bytes); assert_eq!( e.markdown, - "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| x | y |\n| --- | --- |\n" + "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| A | B |\n| --- | --- |\n| x | y |\n" ); assert_eq!(e.stats.sheets, Some(2)); + assert_eq!(e.stats.rows, Some(2)); } #[test] @@ -275,3 +276,118 @@ fn xls_by_content_type() { .unwrap(); assert!(e.markdown.contains("| x |"), "{}", e.markdown); } + +// ─── Single-row sheets, titles, package quirks ────────────────────── + +#[test] +fn single_row_sheet_renders_as_data_under_column_letters() { + // One row starting at column C: header is the real column letters. + let data = r#"only7"#; + let bytes = xlsx_with( + &[Sheet::new("One", vec![])], + &[Some(worksheet_xml(data))], + &[], + ); + let e = extract(&bytes); + assert_eq!( + e.markdown, + "## One\n\n| C | D |\n| --- | --- |\n| only | 7 |\n" + ); + assert_eq!(e.stats.rows, Some(1)); + + // Two rows: the first is still the header. + let bytes = xlsx(&[Sheet::new("Two", vec![vec![S("h")], vec![S("v")]])]); + assert_eq!( + extract(&bytes).markdown, + "## Two\n\n| h |\n| --- |\n| v |\n" + ); +} + +#[test] +fn xlsx_title_from_core_properties() { + let core = br#"Quarterly & Annual"#; + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[("docProps/core.xml", core)], + ); + assert_eq!(extract(&bytes).title.as_deref(), Some("Quarterly & Annual")); + // No core properties: no title. + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("v")]])]); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn xlsx_malformed_core_properties_do_not_fail_extraction() { + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[( + "docProps/core.xml", + b"x".repeat(10_000), "t"); + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("v")]])], + &[], + &[("docProps/core.xml", deep.as_bytes())], + ); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn ods_title_from_meta() { + let content = format!( + r#"{}"#, + ods_string_cell("v") + ); + let meta = br#"ODS Report"#; + let bytes = rezip(&ods(&content), &[], &[("meta.xml", meta)]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Ods); + assert_eq!(e.title.as_deref(), Some("ODS Report")); +} + +#[test] +fn xls_title_from_summary_information() { + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "h"), XlsCell::Label(1, 0, "v")], + }]); + let summary = summary_information(1252, b"Legacy Budget"); + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Xls); + assert_eq!(e.title.as_deref(), Some("Legacy Budget")); + + // Garbage property set: no title, extraction still succeeds. + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", b"\xFE\xFFjunk"), + ]); + let e = extract(&bytes); + assert_eq!(e.title, None); + assert!(e.markdown.contains("| v |")); +} + +#[test] +fn xlsx_without_package_relationships_still_parses() { + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])]); + let bytes = rezip(&bytes, &["_rels/.rels"], &[]); + let e = extract(&bytes); + assert_eq!(e.markdown, "## S\n\n| h |\n| --- |\n| v |\n"); +} + +#[test] +fn xlsb_content_type_is_not_xlsx() { + let ct = "application/vnd.ms-excel.sheet.binary.macroEnabled.12"; + assert_eq!(Format::from_content_type(ct), Format::Xlsb); + assert_eq!(Format::from_file_name("book.xlsb"), Format::Xlsb); +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index 7e3a825..cc7c292 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -44,13 +44,28 @@ impl Markkit { pub struct Extraction { // #[non_exhaustive] pub markdown: String, - pub title: Option, // DOCX core.xml dc:title, else first Title-style paragraph + pub title: Option, // see "Document titles" below pub format: Format, pub warnings: Vec, // ": " pub stats: ExtractionStats, // pages / sheets / rows / paragraphs / tables (Option each) } ``` +Non-empty `markdown` always ends with exactly one `\n` (POSIX text convention; outputs +concatenate cleanly); an empty document renders `""`. This is intended and stable. + +### Document titles + +| Format | Source | +|---|---| +| DOCX | `docProps/core.xml` `dc:title` (via `_rels/.rels`), else the first `Title`-style paragraph | +| XLSX | `docProps/core.xml` `dc:title` | +| ODS | `meta.xml` `dc:title` | +| XLS | `\u{5}SummaryInformation` property set, `PIDSI_TITLE` (code page aware: 1200 UTF-16, UTF-8, else Latin-1) | + +Whitespace is collapsed, titles are capped at 1024 characters, and a missing or +malformed property part yields `None` without failing the extraction. + `extract_bytes(bytes, content_type)` delegates to `extract_document_sync` and returns only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. @@ -60,12 +75,14 @@ Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. ## Format Detection Priority order (`Format::detect`): -1. Content type (`DocumentHint::content_type`; parameters ignored). Generic types such as +1. Content type (`DocumentHint::content_type`; parameters ignored). XLSB + (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) is matched before the + generic `ms-excel.sheet` → XLSX rule. Generic types such as `application/octet-stream` map to `Unknown` and fall through. 2. File extension (`DocumentHint::file_name`; URL query/fragment ignored). 3. Magic bytes (`Format::sniff`): - `PK\x03\x04` ZIP: ODF `mimetype` entry → ODS; `word/document*` → DOCX; - `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored + `xl/workbook.bin` → XLSB; `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored uncompressed, so no archive parsing is needed for detection). - OLE2 (`D0 CF 11 E0`): UTF-16 `WordDocument` stream → DOC; `Workbook`/`Book` → XLS. - `%PDF-` within the first 1 KiB → PDF. @@ -95,7 +112,11 @@ numbering through the main part's relationships. - One `## ` section per non-empty worksheet, in workbook order, followed by a GFM table. The first non-empty row is the header; fully empty rows and leading empty - columns are dropped. + columns are dropped. A sheet with a single non-empty row has no header: its columns + are labelled with their Excel letters (`| C | D |`) and the row renders as data + (counted in `stats.rows`). +- XLSX packages without `_rels/.rels` (hand-built files) get the standard + officeDocument relationship to `xl/workbook.xml` before calamine opens them. - Cells: numbers without trailing `.0` for integers; booleans `true`/`false`; dates as `YYYY-MM-DD[ HH:MM:SS]`, durations `H:MM:SS`; errors as Excel shows them (`#DIV/0!`). Pipes are escaped (`\|`), line breaks flattened to spaces. diff --git a/knowledge/log.md b/knowledge/log.md index 1b2ff07..d6797ed 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,8 @@ ## 2026-10-26 +* **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. + * **URL fetch hardening**: T3 SSRF is mitigated. `extract_url` blocks non-public addresses by default (URL, resolver and redirect layers), enforces scheme, allowlist, redirect, body and time limits; new invariant S12 and `url_security` tests. Proxy-resolved names are the documented residual. See [Threat Model](security/threat-model.md) and [Security Model](security/security-model.md). * **Dependency trim**: unused dependencies removed (lockfile shrinks by ~460 lines; the `instant`/`paste` unmaintained advisories and their `deny.toml` ignores are gone with the stub bench deps). quick-xml moves to 0.42, one version shared with calamine; markkit's XML walkers now read `&str` names and values. See [Threat Model](security/threat-model.md) T6d and [Maintenance](operations/maintenance.md). * **Inter-block whitespace**: indented HTML lost blank lines between blocks and leaked a leading space into every following line, because whitespace-only text between blocks stayed in the block list as `Text(" ")`. The parser now normalizes block-context lists; golden fixtures, which had recorded the broken output, are regenerated. See [HTML Extraction](formats/html-extraction.md) and [IR Specification](foundations/ir.md). diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 49d78f1..d694522 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -187,7 +187,11 @@ process, or smuggling content past a human reviewer. DIMENSIONS, inverted MULRK column ranges, out-of-range sheet offsets). - XML: depth capped at 256, tree built iteratively, unclosed elements tolerated. - Parsers run under `catch_unwind` (defence in depth; only effective with - `panic = "unwind"` -- the workspace release profile uses `abort`). + `panic = "unwind"`, which the workspace release profile uses so the CLI and the + Python/Node bindings survive a parser panic; embedders building with `abort` lose + this layer). +- XLS `SummaryInformation` title parsing is bounds-checked (1 MiB stream cap, checked + offsets, 1024-property cap); failures yield no title, never an error. #### T6d: XML entity attacks (billion laughs, XXE) - quick-xml never expands DTD-declared or external entities; such references are dropped. @@ -213,6 +217,8 @@ process, or smuggling content past a human reviewer. `spreadsheet_integration::xlsx_sparse_far_cell_does_not_allocate_grid` - `docx_billion_laughs_not_expanded`, `docx_external_entity_not_resolved`, `docx_deep_nesting_does_not_overflow_stack` +- `spreadsheet::tests::summary_title_rejects_malformed`, + `spreadsheet_integration::xlsx_malformed_core_properties_do_not_fail_extraction` - `truncated_fixtures_never_panic`, `random_bytes_never_panic`, `random_container_bytes_never_panic`, `mutated_fixtures_never_panic` - `spreadsheet_integration::xlsx_hidden_sheet_skipped_by_default`, From c913ef07e209dbf33bbefb2d512de25e2f28fd86 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi Date: Mon, 26 Oct 2026 23:53:56 +0000 Subject: [PATCH 2/4] fix(spreadsheet): single-row sheets, workbook titles, XLSB detection - Single-row sheets render the row as data under Excel column letters - Extraction.title from XLSX core.xml, ODS meta.xml, XLS SummaryInformation - XLSB content type / extension / container detected as Format::Xlsb, not Xlsx - XLSX without _rels/.rels gets the standard package relationship - Release profile uses panic = "unwind" so catch_unwind guards work - Document the trailing newline as intended Part of #11 --- CHANGELOG.md | 9 + Cargo.toml | 5 +- crates/markkit/src/extract.rs | 16 +- crates/markkit/src/format.rs | 21 ++ crates/markkit/src/lib.rs | 3 +- crates/markkit/src/spreadsheet.rs | 273 +++++++++++++++++- crates/markkit/tests/common/mod.rs | 61 +++- crates/markkit/tests/document_security.rs | 20 +- .../markkit/tests/spreadsheet_integration.rs | 118 +++++++- knowledge/formats/format-support.md | 30 +- knowledge/log.md | 1 + knowledge/security/threat-model.md | 8 +- 12 files changed, 533 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb6e175..408ca37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- Spreadsheets: a sheet with a single row now renders that row as data under Excel column letters + (`| A | B |`) instead of as a header with no rows. +- Spreadsheets: `Extraction.title` is filled from document properties (XLSX `docProps/core.xml`, + ODS `meta.xml`, XLS `SummaryInformation`). +- `Format::from_content_type` detects XLSB (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) + as the new `Format::Xlsb` instead of `Xlsx`; `.xlsb` files and `xl/workbook.bin` containers too. +- XLSX packages missing `_rels/.rels` no longer fail inside calamine. +- Release builds use `panic = "unwind"`, so a parser panic becomes an error instead of aborting the + CLI or the host Python/Node process. - Indented HTML (most real web pages) now keeps blank lines between blocks; previously headings, paragraphs, lists and tables after the first ran together with a leading space (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. diff --git a/Cargo.toml b/Cargo.toml index ab34c1a..4f0e07c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -73,5 +73,8 @@ proptest = "1" [profile.release] lto = "thin" codegen-units = 1 -panic = "abort" +# Unwind (not abort): `extract_document` wraps third-party parsers in +# `catch_unwind`, and PyO3/NAPI turn panics into exceptions. Abort would let one +# malformed document kill the CLI or the host Python/Node process. +panic = "unwind" strip = true diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index 7ce0a14..2c3c9c7 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -8,10 +8,12 @@ use crate::format::{DocumentHint, Format}; #[derive(Debug, Clone, PartialEq, Serialize)] #[non_exhaustive] pub struct Extraction { - /// Rendered Markdown. + /// Rendered Markdown. Non-empty output always ends with exactly one `\n` + /// (POSIX text convention, safe to concatenate); empty documents render `""`. pub markdown: String, - /// Document title when known (HTML `og:title` / JSON-LD / ``; - /// DOCX core properties or `Title` style). + /// Document title when known: HTML `og:title` / JSON-LD / `<title>`; DOCX + /// core properties or `Title` style; XLSX `docProps/core.xml`, ODS + /// `meta.xml`, XLS `SummaryInformation`. pub title: Option<String>, /// Format the document was parsed as. pub format: Format, @@ -81,7 +83,8 @@ impl Extraction { } /// Run a parser, converting a panic into an error (defence in depth for -/// third-party parsers; only effective when built with `panic = "unwind"`). +/// third-party parsers; only effective when built with `panic = "unwind"`, which +/// markkit's own release profile uses). #[cfg(any(feature = "docx", feature = "spreadsheet"))] fn guarded<T>( what: &str, @@ -208,7 +211,9 @@ impl Markkit { Format::Html => self.extract_html_document(&String::from_utf8_lossy(bytes)), Format::Pdf => self.extract_pdf_sync(bytes), Format::Docx => self.extract_docx(bytes), - Format::Xlsx | Format::Xls | Format::Ods => self.extract_spreadsheet(bytes), + Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => { + self.extract_spreadsheet(bytes) + } Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -266,6 +271,7 @@ impl Markkit { })?; let mut renderer = crate::render::MarkdownRenderer::new(); let mut e = Extraction::new(renderer.render(&out.doc), out.format); + e.title = out.title; e.warnings = out.warnings; e.stats.sheets = Some(out.sheets); e.stats.rows = Some(out.rows); diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index cf632a7..509604c 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -22,6 +22,8 @@ pub enum Format { Doc, /// Office Open XML spreadsheet (`.xlsx`, `.xlsm`, `.xltx`). Xlsx, + /// Excel binary workbook (`.xlsb`). + Xlsb, /// Legacy Excel 97-2003 binary (`.xls`). Xls, /// OpenDocument spreadsheet (`.ods`). @@ -76,6 +78,7 @@ impl Format { Format::Docx => "docx", Format::Doc => "doc", Format::Xlsx => "xlsx", + Format::Xlsb => "xlsb", Format::Xls => "xls", Format::Ods => "ods", Format::Pptx => "pptx", @@ -98,6 +101,8 @@ impl Format { Format::Docx } else if ct.contains("msword") { Format::Doc + } else if ct.contains("ms-excel.sheet.binary") { + Format::Xlsb } else if ct.contains("spreadsheetml") || ct.contains("ms-excel.sheet") { Format::Xlsx } else if ct.contains("ms-excel") { @@ -125,6 +130,7 @@ impl Format { "docx" | "docm" | "dotx" | "dotm" => Format::Docx, "doc" | "dot" => Format::Doc, "xlsx" | "xlsm" | "xltx" | "xltm" => Format::Xlsx, + "xlsb" => Format::Xlsb, "xls" | "xlt" => Format::Xls, "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, @@ -198,6 +204,8 @@ fn sniff_zip(bytes: &[u8]) -> Format { Format::Ods } else if contains(bytes, b"word/document") { Format::Docx + } else if contains(bytes, b"xl/workbook.bin") { + Format::Xlsb } else if contains(bytes, b"xl/workbook.") { Format::Xlsx } else if contains(bytes, b"ppt/presentation.") { @@ -245,6 +253,18 @@ mod tests { ); assert_eq!(Format::from_content_type("application/pdf"), Format::Pdf); assert_eq!(Format::from_content_type("text/plain"), Format::PlainText); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.binary.macroEnabled.12"), + Format::Xlsb + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.macroEnabled.12"), + Format::Xlsx + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel"), + Format::Xls + ); } #[test] @@ -252,6 +272,7 @@ mod tests { assert_eq!(Format::from_extension("html"), Format::Html); assert_eq!(Format::from_extension("PDF"), Format::Pdf); assert_eq!(Format::from_extension("docx"), Format::Docx); + assert_eq!(Format::from_extension("XLSB"), Format::Xlsb); } #[test] diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index e68e111..ff97b56 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -60,7 +60,8 @@ pub mod render; mod spreadsheet; #[cfg(feature = "tool")] pub mod tool; -#[cfg(feature = "docx")] +#[cfg(any(feature = "docx", feature = "spreadsheet"))] +#[cfg_attr(not(feature = "docx"), allow(dead_code))] // spreadsheet uses only `find`/`text` mod xml; pub use config::MarkkitBuilder; diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index 2b04212..cca4777 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -34,6 +34,7 @@ use crate::render::escape_table_cell; pub(crate) struct SheetOutput { pub doc: Document, pub format: Format, + pub title: Option<String>, pub warnings: Vec<String>, pub sheets: usize, pub rows: usize, @@ -63,7 +64,8 @@ enum Container { } pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput, Error> { - let container = if bytes.starts_with(b"PK") { + let mut patched = None; + let (container, title) = if bytes.starts_with(b"PK") { let mut zip = ZipGuard::open(bytes, config).map_err(archive_err)?; let has = |zip: &ZipGuard<'_>, name: &str| zip.names().any(|n| n.eq_ignore_ascii_case(name)); @@ -79,20 +81,29 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput return Err(err("ZIP container is not an XLSX or ODS workbook")); }; zip.validate_all().map_err(archive_err)?; - if let Container::Ods = container { + let title = if let Container::Ods = container { let content = zip .read("content.xml") .map_err(archive_err)? .ok_or_else(|| err("ODS is missing content.xml"))?; prescan_ods(&content, config.max_spreadsheet_cells)?; - } - container + read_dc_title(&mut zip, "meta.xml")? + } else { + if !has(&zip, "_rels/.rels") { + patched = Some(with_package_rels(&mut zip)?); + } + read_dc_title(&mut zip, "docProps/core.xml")? + }; + (container, title) } else if bytes.starts_with(OLE2_MAGIC) { - prescan_xls(bytes, config)?; - Container::Xls + let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)) + .map_err(|e| err(format!("invalid XLS container: {e}")))?; + prescan_xls(&mut comp, config)?; + (Container::Xls, xls_title(&mut comp)) } else { return Err(err("not a spreadsheet (expected XLSX, XLS or ODS)")); }; + let bytes = patched.as_deref().unwrap_or(bytes); let mut out = Collector::new(config); match container { @@ -116,19 +127,19 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput Err(e) => out.warn(format!("sheet_error: sheet '{name}': {e}")), } } - out.finish(Format::Xlsx) + out.finish(Format::Xlsx, title) } Container::Ods => { let mut wb: Ods<_> = Ods::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Ods) + out.finish(Format::Ods, title) } Container::Xls => { let mut wb: Xls<_> = Xls::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Xls) + out.finish(Format::Xls, title) } } } @@ -217,6 +228,19 @@ fn fmt_ref(d: &DataRef<'_>) -> String { // ─── Grid collection (bounded) ────────────────────────────────────── +/// Excel column letters for a 0-based index (`0` → `A`, `26` → `AA`). +fn col_name(c: u32) -> String { + let mut n = u64::from(c) + 1; + let mut out = Vec::new(); + while n > 0 { + n -= 1; + out.push(b'A' + (n % 26) as u8); + n /= 26; + } + out.reverse(); + String::from_utf8(out).unwrap_or_default() +} + struct SheetTable { header: Vec<String>, rows: Vec<Vec<String>>, @@ -298,10 +322,19 @@ impl Grid { } row }); - let header = rows.next()?; + let first = rows.next()?; + let rest: Vec<Vec<String>> = rows.collect(); + // GFM tables need a header row. A sheet with a single row is data, not a + // header: label its columns A, B, … so the row renders as a data row. + let (header, rows) = if rest.is_empty() { + let letters = (min_col..min_col + width as u32).map(col_name).collect(); + (letters, vec![first]) + } else { + (first, rest) + }; Some(SheetTable { header, - rows: rows.collect(), + rows, truncated_rows: self.truncated_rows, truncated_cols: self.truncated_cols, }) @@ -396,10 +429,11 @@ impl<'c> Collector<'c> { }); } - fn finish(self, format: Format) -> Result<SheetOutput, Error> { + fn finish(self, format: Format, title: Option<String>) -> Result<SheetOutput, Error> { Ok(SheetOutput { doc: Document::new(self.nodes), format, + title, warnings: self.warnings, sheets: self.sheets, rows: self.rows, @@ -407,6 +441,128 @@ impl<'c> Collector<'c> { } } +// ─── Package fixes and document title ─────────────────────────────── + +/// Longest title kept (characters); longer titles are cut. +const MAX_TITLE_CHARS: usize = 1024; + +/// calamine locates the workbook through `_rels/.rels` and fails without it. +/// Rebuild the (already validated) archive with the standard relationship to +/// `xl/workbook.xml`; entries are copied raw, without recompression. +fn with_package_rels(zip: &mut ZipGuard<'_>) -> Result<Vec<u8>, Error> { + const RELS: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#; + let zerr = |e: zip::result::ZipError| err(format!("cannot rebuild XLSX package: {e}")); + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + for i in 0..zip.archive.len() { + let entry = zip.archive.by_index_raw(i).map_err(zerr)?; + w.raw_copy_file(entry).map_err(zerr)?; + } + let opts = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + w.start_file("_rels/.rels", opts).map_err(zerr)?; + std::io::Write::write_all(&mut w, RELS.as_bytes()) + .map_err(|e| err(format!("cannot rebuild XLSX package: {e}")))?; + Ok(w.finish().map_err(zerr)?.into_inner()) +} + +fn clean_title(raw: &str) -> Option<String> { + let t: String = raw + .split_whitespace() + .collect::<Vec<_>>() + .join(" ") + .chars() + .take(MAX_TITLE_CHARS) + .collect(); + (!t.is_empty()).then_some(t) +} + +/// `dc:title` from OOXML `docProps/core.xml` or ODF `meta.xml`. Best effort: +/// a missing or malformed part yields `None`; only archive limits are errors. +fn read_dc_title(zip: &mut ZipGuard<'_>, part: &str) -> Result<Option<String>, Error> { + let Some(xml) = zip.read(part).map_err(archive_err)? else { + return Ok(None); + }; + Ok(dc_title(&xml)) +} + +fn dc_title(xml: &[u8]) -> Option<String> { + let root = crate::xml::parse(xml).ok()?; + clean_title(&root.find("title")?.text()) +} + +/// Title (PIDSI_TITLE) from the XLS `\u{5}SummaryInformation` property set +/// ([MS-OLEPS]). Best effort: anything malformed yields `None`. +fn xls_title<F: Read + Seek>(comp: &mut cfb::CompoundFile<F>) -> Option<String> { + const CAP: u64 = 1024 * 1024; + let mut data = Vec::new(); + comp.open_stream("/\u{5}SummaryInformation") + .ok()? + .take(CAP) + .read_to_end(&mut data) + .ok()?; + summary_title(&data) +} + +fn summary_title(d: &[u8]) -> Option<String> { + const PID_CODEPAGE: u32 = 1; + const PID_TITLE: u32 = 2; + const VT_I2: u32 = 0x0002; + const VT_LPSTR: u32 = 0x001E; + const VT_LPWSTR: u32 = 0x001F; + if read_u16(d, 0)? != 0xFFFE || read_u32(d, 24)? == 0 { + return None; + } + let set = read_u32(d, 44)? as usize; + let count = read_u32(d, set.checked_add(4)?)? as usize; + let mut codepage = None; + let mut title_at = None; + for i in 0..count.min(1024) { + let entry = set.checked_add(8 + i * 8)?; + let (pid, off) = (read_u32(d, entry)?, read_u32(d, entry + 4)? as usize); + let at = set.checked_add(off)?; + match pid { + PID_CODEPAGE if read_u32(d, at)? & 0xFFFF == VT_I2 => { + codepage = Some(read_u16(d, at + 4)?); + } + PID_TITLE => title_at = Some(at), + _ => {} + } + } + let at = title_at?; + let typ = read_u32(d, at)? & 0xFFFF; + let n = read_u32(d, at + 4)? as usize; + let start = at + 8; + let raw = match typ { + VT_LPWSTR => { + let bytes = d.get(start..start.checked_add(n.checked_mul(2)?)?)?; + let units: Vec<u16> = bytes + .chunks_exact(2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .collect(); + String::from_utf16_lossy(&units) + } + VT_LPSTR => { + let bytes = d.get(start..start.checked_add(n)?)?; + match codepage { + Some(1200) => { + let units: Vec<u16> = bytes + .chunks_exact(2) + .map(|b| u16::from_le_bytes([b[0], b[1]])) + .collect(); + String::from_utf16_lossy(&units) + } + // UTF-8 when valid, otherwise Latin-1 (close to Windows-1252). + _ => match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), + }, + } + } + _ => return None, + }; + clean_title(raw.trim_end_matches('\0')) +} + // ─── XLS pre-scan (BIFF records) ──────────────────────────────────── fn read_u16(d: &[u8], at: usize) -> Option<u16> { @@ -430,9 +586,10 @@ fn records(stream: &[u8]) -> impl Iterator<Item = (u16, &[u8])> { }) } -fn prescan_xls(bytes: &[u8], config: &MarkkitBuilder) -> Result<(), Error> { - let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)) - .map_err(|e| err(format!("invalid XLS container: {e}")))?; +fn prescan_xls<F: Read + Seek>( + comp: &mut cfb::CompoundFile<F>, + config: &MarkkitBuilder, +) -> Result<(), Error> { let name = ["/Workbook", "/Book", "/WORKBOOK", "/BOOK"] .into_iter() .find(|n| comp.is_stream(n)) @@ -706,6 +863,92 @@ mod tests { assert!(t.truncated_rows); } + #[test] + fn column_names() { + assert_eq!(col_name(0), "A"); + assert_eq!(col_name(25), "Z"); + assert_eq!(col_name(26), "AA"); + assert_eq!(col_name(16_383), "XFD"); + assert_eq!(col_name(701), "ZZ"); + assert_eq!(col_name(702), "AAA"); + assert!(!col_name(u32::MAX).is_empty()); + } + + #[test] + fn single_row_becomes_data_under_column_letters() { + let cfg = MarkkitBuilder::default(); + let mut g = Grid::new(&cfg); + g.add(4, 2, "x".into()); + g.add(4, 3, "y".into()); + let t = g.into_table().unwrap(); + assert_eq!(t.header, vec!["C", "D"]); + assert_eq!(t.rows, vec![vec!["x".to_string(), "y".to_string()]]); + } + + #[test] + fn dc_title_reads_and_normalizes() { + let core = br#"<cp:coreProperties xmlns:cp="c" xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title> Q3 + Budget & Plan </dc:title></cp:coreProperties>"#; + assert_eq!(dc_title(core).as_deref(), Some("Q3 Budget & Plan")); + assert_eq!(dc_title(b"<a><dc:title> </dc:title></a>"), None); + assert_eq!(dc_title(b"<a></a>"), None); + assert_eq!(dc_title(b"\xff\xfe<<"), None); + let long = format!("<title>{}", "x".repeat(5000)); + assert_eq!(dc_title(long.as_bytes()).unwrap().len(), MAX_TITLE_CHARS); + } + + fn summary_info(codepage: u16, typ: u32, value: &[u8], count: u32) -> Vec { + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + // Property set: size, count, 2 × (pid, offset), codepage, title. + let mut set = Vec::new(); + set.extend_from_slice(&0u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&1u32.to_le_bytes()); + set.extend_from_slice(&24u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&32u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&u32::from(codepage).to_le_bytes()); + set.extend_from_slice(&typ.to_le_bytes()); + set.extend_from_slice(&count.to_le_bytes()); + set.extend_from_slice(value); + d.extend_from_slice(&set); + d + } + + #[test] + fn summary_title_decodes_codepages() { + let d = summary_info(1252, 0x1E, b"Sales\0", 6); + assert_eq!(summary_title(&d).as_deref(), Some("Sales")); + let d = summary_info(1252, 0x1E, b"Caf\xe9\0", 5); + assert_eq!(summary_title(&d).as_deref(), Some("Café")); + let d = summary_info(65001, 0x1E, "Звіт\0".as_bytes(), 9); + assert_eq!(summary_title(&d).as_deref(), Some("Звіт")); + let wide: Vec = "Hi\0".encode_utf16().flat_map(u16::to_le_bytes).collect(); + let d = summary_info(1200, 0x1F, &wide, 3); + assert_eq!(summary_title(&d).as_deref(), Some("Hi")); + } + + #[test] + fn summary_title_rejects_malformed() { + assert_eq!(summary_title(b""), None); + assert_eq!(summary_title(&[0xFE, 0xFF, 0, 0]), None); + // Length pointing past the stream. + let d = summary_info(1252, 0x1E, b"abc", u32::MAX); + assert_eq!(summary_title(&d), None); + // Property offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[48 + 20..48 + 24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + // Section offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[44..48].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + } + #[test] fn biff_scan_rejects_inverted_mulrk() { let mut s = Vec::new(); diff --git a/crates/markkit/tests/common/mod.rs b/crates/markkit/tests/common/mod.rs index 47bbd2b..268d897 100644 --- a/crates/markkit/tests/common/mod.rs +++ b/crates/markkit/tests/common/mod.rs @@ -416,15 +416,70 @@ pub fn biff8_workbook(sheets: &[XlsSheet]) -> Vec { /// Wrap a workbook stream into an OLE2 compound file. pub fn xls_from_stream(stream: &[u8]) -> Vec { + ole2(&[("/Workbook", stream)]) +} + +/// Build an OLE2 compound file from `(path, data)` streams. +pub fn ole2(streams: &[(&str, &[u8])]) -> Vec { let mut comp = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); - { - let mut s = comp.create_stream("/Workbook").unwrap(); - s.write_all(stream).unwrap(); + for (path, data) in streams { + let mut s = comp.create_stream(path).unwrap(); + s.write_all(data).unwrap(); } comp.flush().unwrap(); comp.into_inner().into_inner() } +/// `\u{5}SummaryInformation` property set with a code page and a VT_LPSTR title. +pub fn summary_information(codepage: u16, title: &[u8]) -> Vec { + let mut value = title.to_vec(); + value.push(0); + let mut set = Vec::new(); + for v in [ + 0u32, + 2, + 1, + 24, + 2, + 32, + 2, + u32::from(codepage), + 0x1E, + value.len() as u32, + ] { + set.extend_from_slice(&v.to_le_bytes()); + } + set.extend_from_slice(&value); + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + d.extend_from_slice(&set); + d +} + +/// Rebuild a ZIP without the `drop` entries and with `add` appended. +pub fn rezip(bytes: &[u8], drop: &[&str], add: &[(&str, &[u8])]) -> Vec { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + let mut entries = Vec::new(); + for i in 0..zip.len() { + let mut f = zip.by_index(i).unwrap(); + let name = f.name().unwrap().to_string(); + if drop.contains(&name.as_str()) { + continue; + } + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut f, &mut data).unwrap(); + entries.push((name, data)); + } + let mut refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(n, d)| (n.as_str(), d.as_slice())) + .collect(); + refs.extend_from_slice(add); + zip_bytes(&refs) +} + pub fn xls(sheets: &[XlsSheet]) -> Vec { xls_from_stream(&biff8_workbook(sheets)) } diff --git a/crates/markkit/tests/document_security.rs b/crates/markkit/tests/document_security.rs index 693ea25..ce64334 100644 --- a/crates/markkit/tests/document_security.rs +++ b/crates/markkit/tests/document_security.rs @@ -319,6 +319,24 @@ fn valid_fixtures() -> Vec> { dims: Some((0, 1, 0, 2)), cells: vec![XlsCell::Label(0, 0, "a"), XlsCell::Num(0, 1, 2.0)], }])); + // Title sources: XLS SummaryInformation, XLSX core.xml + no `_rels/.rels`. + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "a")], + }]); + let summary = summary_information(1252, b"Title"); + v.push(ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ])); + let core = b"T"; + let book = xlsx_with( + &[Sheet::new("S", vec![vec![Cell::S("a")]])], + &[], + &[("docProps/core.xml", core)], + ); + v.push(rezip(&book, &["_rels/.rels"], &[])); } v } @@ -379,7 +397,7 @@ proptest! { /// Byte flips in valid documents never panic. #[test] fn mutated_fixtures_never_panic( - idx in 0usize..4, + idx in 0usize..6, flips in proptest::collection::vec((any::(), any::()), 1..16), ) { let fixtures = valid_fixtures(); diff --git a/crates/markkit/tests/spreadsheet_integration.rs b/crates/markkit/tests/spreadsheet_integration.rs index d976897..111e8c7 100644 --- a/crates/markkit/tests/spreadsheet_integration.rs +++ b/crates/markkit/tests/spreadsheet_integration.rs @@ -50,9 +50,10 @@ fn xlsx_multiple_sheets_and_empty_sheet_skipped() { let e = extract(&bytes); assert_eq!( e.markdown, - "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| x | y |\n| --- | --- |\n" + "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| A | B |\n| --- | --- |\n| x | y |\n" ); assert_eq!(e.stats.sheets, Some(2)); + assert_eq!(e.stats.rows, Some(2)); } #[test] @@ -275,3 +276,118 @@ fn xls_by_content_type() { .unwrap(); assert!(e.markdown.contains("| x |"), "{}", e.markdown); } + +// ─── Single-row sheets, titles, package quirks ────────────────────── + +#[test] +fn single_row_sheet_renders_as_data_under_column_letters() { + // One row starting at column C: header is the real column letters. + let data = r#"only7"#; + let bytes = xlsx_with( + &[Sheet::new("One", vec![])], + &[Some(worksheet_xml(data))], + &[], + ); + let e = extract(&bytes); + assert_eq!( + e.markdown, + "## One\n\n| C | D |\n| --- | --- |\n| only | 7 |\n" + ); + assert_eq!(e.stats.rows, Some(1)); + + // Two rows: the first is still the header. + let bytes = xlsx(&[Sheet::new("Two", vec![vec![S("h")], vec![S("v")]])]); + assert_eq!( + extract(&bytes).markdown, + "## Two\n\n| h |\n| --- |\n| v |\n" + ); +} + +#[test] +fn xlsx_title_from_core_properties() { + let core = br#"Quarterly & Annual"#; + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[("docProps/core.xml", core)], + ); + assert_eq!(extract(&bytes).title.as_deref(), Some("Quarterly & Annual")); + // No core properties: no title. + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("v")]])]); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn xlsx_malformed_core_properties_do_not_fail_extraction() { + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[( + "docProps/core.xml", + b"x".repeat(10_000), "t"); + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("v")]])], + &[], + &[("docProps/core.xml", deep.as_bytes())], + ); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn ods_title_from_meta() { + let content = format!( + r#"{}"#, + ods_string_cell("v") + ); + let meta = br#"ODS Report"#; + let bytes = rezip(&ods(&content), &[], &[("meta.xml", meta)]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Ods); + assert_eq!(e.title.as_deref(), Some("ODS Report")); +} + +#[test] +fn xls_title_from_summary_information() { + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "h"), XlsCell::Label(1, 0, "v")], + }]); + let summary = summary_information(1252, b"Legacy Budget"); + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Xls); + assert_eq!(e.title.as_deref(), Some("Legacy Budget")); + + // Garbage property set: no title, extraction still succeeds. + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", b"\xFE\xFFjunk"), + ]); + let e = extract(&bytes); + assert_eq!(e.title, None); + assert!(e.markdown.contains("| v |")); +} + +#[test] +fn xlsx_without_package_relationships_still_parses() { + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])]); + let bytes = rezip(&bytes, &["_rels/.rels"], &[]); + let e = extract(&bytes); + assert_eq!(e.markdown, "## S\n\n| h |\n| --- |\n| v |\n"); +} + +#[test] +fn xlsb_content_type_is_not_xlsx() { + let ct = "application/vnd.ms-excel.sheet.binary.macroEnabled.12"; + assert_eq!(Format::from_content_type(ct), Format::Xlsb); + assert_eq!(Format::from_file_name("book.xlsb"), Format::Xlsb); +} diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index f1f915b..0ae6deb 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -44,7 +44,7 @@ impl Markkit { pub struct Extraction { // #[non_exhaustive] pub markdown: String, - pub title: Option, // HTML og:title / JSON-LD / ; DOCX core.xml dc:title, else Title style + pub title: Option<String>, // see "Document titles" below pub format: Format, pub warnings: Vec<String>, // "<code>: <message>" pub stats: ExtractionStats, // pages / sheets / rows / paragraphs / tables (Option each) @@ -52,6 +52,22 @@ pub struct Extraction { // #[non_exhaustive] } ``` +Non-empty `markdown` always ends with exactly one `\n` (POSIX text convention; outputs +concatenate cleanly); an empty document renders `""`. This is intended and stable. + +### Document titles + +| Format | Source | +|---|---| +| HTML | `og:title` / JSON-LD / `<title>` (see [HTML Extraction](html-extraction.md)) | +| DOCX | `docProps/core.xml` `dc:title` (via `_rels/.rels`), else the first `Title`-style paragraph | +| XLSX | `docProps/core.xml` `dc:title` | +| ODS | `meta.xml` `dc:title` | +| XLS | `\u{5}SummaryInformation` property set, `PIDSI_TITLE` (code page aware: 1200 UTF-16, UTF-8, else Latin-1) | + +Whitespace is collapsed, titles are capped at 1024 characters, and a missing or +malformed property part yields `None` without failing the extraction. + `extract_bytes(bytes, content_type)` delegates to `extract_document_sync` and returns only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. @@ -61,12 +77,14 @@ Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. ## Format Detection Priority order (`Format::detect`): -1. Content type (`DocumentHint::content_type`; parameters ignored). Generic types such as +1. Content type (`DocumentHint::content_type`; parameters ignored). XLSB + (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) is matched before the + generic `ms-excel.sheet` → XLSX rule. Generic types such as `application/octet-stream` map to `Unknown` and fall through. 2. File extension (`DocumentHint::file_name`; URL query/fragment ignored). 3. Magic bytes (`Format::sniff`): - `PK\x03\x04` ZIP: ODF `mimetype` entry → ODS; `word/document*` → DOCX; - `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored + `xl/workbook.bin` → XLSB; `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored uncompressed, so no archive parsing is needed for detection). - OLE2 (`D0 CF 11 E0`): UTF-16 `WordDocument` stream → DOC; `Workbook`/`Book` → XLS. - `%PDF-` within the first 1 KiB → PDF. @@ -96,7 +114,11 @@ numbering through the main part's relationships. - One `## <Sheet name>` section per non-empty worksheet, in workbook order, followed by a GFM table. The first non-empty row is the header; fully empty rows and leading empty - columns are dropped. + columns are dropped. A sheet with a single non-empty row has no header: its columns + are labelled with their Excel letters (`| C | D |`) and the row renders as data + (counted in `stats.rows`). +- XLSX packages without `_rels/.rels` (hand-built files) get the standard + officeDocument relationship to `xl/workbook.xml` before calamine opens them. - Cells: numbers without trailing `.0` for integers; booleans `true`/`false`; dates as `YYYY-MM-DD[ HH:MM:SS]`, durations `H:MM:SS`; errors as Excel shows them (`#DIV/0!`). Pipes are escaped (`\|`), line breaks flattened to spaces. diff --git a/knowledge/log.md b/knowledge/log.md index 38c2dc7..f639756 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -2,6 +2,7 @@ ## 2026-10-26 +* **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. * **Page metadata**: HTML `extract_document` now fills `title` and a new `Extraction::metadata` (description, author, published, language, canonical URL, site name, image) from `<meta>`, Open Graph, `<link rel=canonical>` and bounded JSON-LD. Precedence table in [HTML Extraction](formats/html-extraction.md); scheme filter and JSON-LD bounds in [Threat Model](security/threat-model.md) T1 / T2f. * **IR v2**: footnotes, definition lists, task lists, figures, table alignment and spans; `Document.ir_version` (`IR_VERSION = 2`), `#[non_exhaustive]` IR types, `Node::is_block`/`is_inline`. HTML maps GitHub/Pandoc footnotes, `<dl>`, checkbox items, captioned figures, `colspan`/`rowspan`/alignment and row-header `<th>`. New threat T2e and invariant S13 (span grid budget, footnote label sanitizing). See [IR Specification](foundations/ir.md), [HTML Extraction](formats/html-extraction.md), [Threat Model](security/threat-model.md). * **Agent tool**: new [Tool Contract](integrations/tool-contract.md) concept in a new `integrations/` domain. `MarkkitTool` (opt-in `tool` feature) follows the Everruns toolkit-library contract; tool arguments are a new untrusted surface in the [Threat Model](security/threat-model.md). diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 66496af..80d3ffb 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -205,7 +205,11 @@ process, or smuggling content past a human reviewer. DIMENSIONS, inverted MULRK column ranges, out-of-range sheet offsets). - XML: depth capped at 256, tree built iteratively, unclosed elements tolerated. - Parsers run under `catch_unwind` (defence in depth; only effective with - `panic = "unwind"` -- the workspace release profile uses `abort`). + `panic = "unwind"`, which the workspace release profile uses so the CLI and the + Python/Node bindings survive a parser panic; embedders building with `abort` lose + this layer). +- XLS `SummaryInformation` title parsing is bounds-checked (1 MiB stream cap, checked + offsets, 1024-property cap); failures yield no title, never an error. #### T6d: XML entity attacks (billion laughs, XXE) - quick-xml never expands DTD-declared or external entities; such references are dropped. @@ -231,6 +235,8 @@ process, or smuggling content past a human reviewer. `spreadsheet_integration::xlsx_sparse_far_cell_does_not_allocate_grid` - `docx_billion_laughs_not_expanded`, `docx_external_entity_not_resolved`, `docx_deep_nesting_does_not_overflow_stack` +- `spreadsheet::tests::summary_title_rejects_malformed`, + `spreadsheet_integration::xlsx_malformed_core_properties_do_not_fail_extraction` - `truncated_fixtures_never_panic`, `random_bytes_never_panic`, `random_container_bytes_never_panic`, `mutated_fixtures_never_panic` - `spreadsheet_integration::xlsx_hidden_sheet_skipped_by_default`, From 7322499159afd6df97e4d0a1fb312358e4368686 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi <mike@chaliy.name> Date: Tue, 27 Oct 2026 00:03:23 +0000 Subject: [PATCH 3/4] fix(spreadsheet): satisfy clippy chunks_exact_to_as_chunks --- crates/markkit/src/spreadsheet.rs | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index cca4777..8cb5787 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -503,6 +503,13 @@ fn xls_title<F: Read + Seek>(comp: &mut cfb::CompoundFile<F>) -> Option<String> summary_title(&data) } +/// Decode UTF-16LE (lossy; a trailing odd byte is ignored). +fn utf16le(bytes: &[u8]) -> String { + let (pairs, _) = bytes.as_chunks::<2>(); + let units: Vec<u16> = pairs.iter().map(|b| u16::from_le_bytes(*b)).collect(); + String::from_utf16_lossy(&units) +} + fn summary_title(d: &[u8]) -> Option<String> { const PID_CODEPAGE: u32 = 1; const PID_TITLE: u32 = 2; @@ -534,23 +541,12 @@ fn summary_title(d: &[u8]) -> Option<String> { let start = at + 8; let raw = match typ { VT_LPWSTR => { - let bytes = d.get(start..start.checked_add(n.checked_mul(2)?)?)?; - let units: Vec<u16> = bytes - .chunks_exact(2) - .map(|b| u16::from_le_bytes([b[0], b[1]])) - .collect(); - String::from_utf16_lossy(&units) + utf16le(d.get(start..start.checked_add(n.checked_mul(2)?)?)?) } VT_LPSTR => { let bytes = d.get(start..start.checked_add(n)?)?; match codepage { - Some(1200) => { - let units: Vec<u16> = bytes - .chunks_exact(2) - .map(|b| u16::from_le_bytes([b[0], b[1]])) - .collect(); - String::from_utf16_lossy(&units) - } + Some(1200) => utf16le(bytes), // UTF-8 when valid, otherwise Latin-1 (close to Windows-1252). _ => match std::str::from_utf8(bytes) { Ok(s) => s.to_string(), From 86b7104c3bd9c9ee7367f95fef285bb6a4cc3865 Mon Sep 17 00:00:00 2001 From: Mykhailo Chalyi <mike@chaliy.name> Date: Tue, 27 Oct 2026 00:32:25 +0000 Subject: [PATCH 4/4] chore(knowledge): tidy log after merge --- knowledge/log.md | 1 - 1 file changed, 1 deletion(-) diff --git a/knowledge/log.md b/knowledge/log.md index 71fdff0..440e1e5 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -4,7 +4,6 @@ * **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. * **Node options parity**: [Node.js Package](runtimes/node-package.md) functions take a trailing options object (all builder fields plus URL policy) and gain `extractDocument`/`extractDocumentSync`; the parity table now covers Python, Node and WASM. - * **WASM package**: new [WASM Package](runtimes/wasm-package.md) concept for `@everruns/markkit-wasm` (wasm-bindgen, single-threaded, no URL fetching), CI job `wasm` gating `Check`, and `publish-wasm.yml`. [Release Process](operations/release-process.md) bumps both npm `package.json` versions. * **Ship contract**: "ship" now ends with the merge commit's `main` CI green, not at merge; the `ship` skill gains existing-PR, auto-merge, and post-merge verification steps aligned with bashkit. See [Maintenance](operations/maintenance.md). * **IR provenance**: optional per-block `Source` (HTML XPath, DOCX body index, sheet + A1 range, page) behind `track_sources`; `parse_document` / `ParsedDocument` expose the IR for every format. XPath sibling indexes are cached per parent (`html_security::source_tracking_many_siblings_is_linear`). See [IR Specification](foundations/ir.md) and [Markkit Architecture](foundations/architecture.md).