diff --git a/CHANGELOG.md b/CHANGELOG.md index 620b9cc..3afb309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Fixed +- Spreadsheets: a sheet with a single row now renders that row as data under Excel column letters + (`| A | B |`) instead of as a header with no rows. +- Spreadsheets: `Extraction.title` is filled from document properties (XLSX `docProps/core.xml`, + ODS `meta.xml`, XLS `SummaryInformation`). +- `Format::from_content_type` detects XLSB (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) + as the new `Format::Xlsb` instead of `Xlsx`; `.xlsb` files and `xl/workbook.bin` containers too. +- XLSX packages missing `_rels/.rels` no longer fail inside calamine. +- Release builds use `panic = "unwind"`, so a parser panic becomes an error instead of aborting the + CLI or the host Python/Node process. - Indented HTML (most real web pages) now keeps blank lines between blocks; previously headings, paragraphs, lists and tables after the first ran together with a leading space (`# A\n B` instead of `# A\n\nB`), which also broke headings and tables as markdown. diff --git a/Cargo.toml b/Cargo.toml index 62dc15f..ebd7d17 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,10 @@ proptest = "1" [profile.release] lto = "thin" codegen-units = 1 -panic = "abort" +# Unwind (not abort): `extract_document` wraps third-party parsers in +# `catch_unwind`, and PyO3/NAPI turn panics into exceptions. Abort would let one +# malformed document kill the CLI or the host Python/Node process. +panic = "unwind" strip = true # Browser bundle (crates/markkit-wasm/scripts/build.sh): size over speed. diff --git a/crates/markkit/src/extract.rs b/crates/markkit/src/extract.rs index f12a7a7..3a89da4 100644 --- a/crates/markkit/src/extract.rs +++ b/crates/markkit/src/extract.rs @@ -9,10 +9,12 @@ use crate::ir::Document; #[derive(Debug, Clone, PartialEq, Serialize)] #[non_exhaustive] pub struct Extraction { - /// Rendered Markdown. + /// Rendered Markdown. Non-empty output always ends with exactly one `\n` + /// (POSIX text convention, safe to concatenate); empty documents render `""`. pub markdown: String, - /// Document title when known (HTML `og:title` / JSON-LD / ``; - /// DOCX core properties or `Title` style). + /// Document title when known: HTML `og:title` / JSON-LD / `<title>`; DOCX + /// core properties or `Title` style; XLSX `docProps/core.xml`, ODS + /// `meta.xml`, XLS `SummaryInformation`. pub title: Option<String>, /// Format the document was parsed as. pub format: Format, @@ -118,7 +120,8 @@ impl ParsedDocument { } /// Run a parser, converting a panic into an error (defence in depth for -/// third-party parsers; only effective when built with `panic = "unwind"`). +/// third-party parsers; only effective when built with `panic = "unwind"`, which +/// markkit's own release profile uses). #[cfg(any(feature = "docx", feature = "spreadsheet"))] fn guarded<T>( what: &str, @@ -266,7 +269,9 @@ impl Markkit { Format::Html => self.parse_html_doc(bytes), Format::Pdf => self.parse_pdf(bytes), Format::Docx => self.parse_docx(bytes), - Format::Xlsx | Format::Xls | Format::Ods => self.parse_spreadsheet(bytes), + Format::Xlsx | Format::Xlsb | Format::Xls | Format::Ods => { + self.parse_spreadsheet(bytes) + } Format::Doc => Err(Error::UnsupportedFormat( "legacy Word 97-2003 (.doc) documents are not supported".into(), )), @@ -328,6 +333,7 @@ impl Markkit { let mut doc = out.doc; self.config.transforms.apply(&mut doc)?; let mut p = ParsedDocument::new(doc, out.format); + p.title = out.title; p.warnings = out.warnings; p.stats.sheets = Some(out.sheets); p.stats.rows = Some(out.rows); diff --git a/crates/markkit/src/format.rs b/crates/markkit/src/format.rs index cf632a7..509604c 100644 --- a/crates/markkit/src/format.rs +++ b/crates/markkit/src/format.rs @@ -22,6 +22,8 @@ pub enum Format { Doc, /// Office Open XML spreadsheet (`.xlsx`, `.xlsm`, `.xltx`). Xlsx, + /// Excel binary workbook (`.xlsb`). + Xlsb, /// Legacy Excel 97-2003 binary (`.xls`). Xls, /// OpenDocument spreadsheet (`.ods`). @@ -76,6 +78,7 @@ impl Format { Format::Docx => "docx", Format::Doc => "doc", Format::Xlsx => "xlsx", + Format::Xlsb => "xlsb", Format::Xls => "xls", Format::Ods => "ods", Format::Pptx => "pptx", @@ -98,6 +101,8 @@ impl Format { Format::Docx } else if ct.contains("msword") { Format::Doc + } else if ct.contains("ms-excel.sheet.binary") { + Format::Xlsb } else if ct.contains("spreadsheetml") || ct.contains("ms-excel.sheet") { Format::Xlsx } else if ct.contains("ms-excel") { @@ -125,6 +130,7 @@ impl Format { "docx" | "docm" | "dotx" | "dotm" => Format::Docx, "doc" | "dot" => Format::Doc, "xlsx" | "xlsm" | "xltx" | "xltm" => Format::Xlsx, + "xlsb" => Format::Xlsb, "xls" | "xlt" => Format::Xls, "ods" => Format::Ods, "pptx" | "pptm" | "ppt" => Format::Pptx, @@ -198,6 +204,8 @@ fn sniff_zip(bytes: &[u8]) -> Format { Format::Ods } else if contains(bytes, b"word/document") { Format::Docx + } else if contains(bytes, b"xl/workbook.bin") { + Format::Xlsb } else if contains(bytes, b"xl/workbook.") { Format::Xlsx } else if contains(bytes, b"ppt/presentation.") { @@ -245,6 +253,18 @@ mod tests { ); assert_eq!(Format::from_content_type("application/pdf"), Format::Pdf); assert_eq!(Format::from_content_type("text/plain"), Format::PlainText); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.binary.macroEnabled.12"), + Format::Xlsb + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel.sheet.macroEnabled.12"), + Format::Xlsx + ); + assert_eq!( + Format::from_content_type("application/vnd.ms-excel"), + Format::Xls + ); } #[test] @@ -252,6 +272,7 @@ mod tests { assert_eq!(Format::from_extension("html"), Format::Html); assert_eq!(Format::from_extension("PDF"), Format::Pdf); assert_eq!(Format::from_extension("docx"), Format::Docx); + assert_eq!(Format::from_extension("XLSB"), Format::Xlsb); } #[test] diff --git a/crates/markkit/src/lib.rs b/crates/markkit/src/lib.rs index 7f9788b..66f11a1 100644 --- a/crates/markkit/src/lib.rs +++ b/crates/markkit/src/lib.rs @@ -62,7 +62,8 @@ mod spreadsheet; #[cfg(feature = "tool")] pub mod tool; pub mod transform; -#[cfg(feature = "docx")] +#[cfg(any(feature = "docx", feature = "spreadsheet"))] +#[cfg_attr(not(feature = "docx"), allow(dead_code))] // spreadsheet uses only `find`/`text` mod xml; pub use config::MarkkitBuilder; diff --git a/crates/markkit/src/spreadsheet.rs b/crates/markkit/src/spreadsheet.rs index 5b1f806..1df6fbe 100644 --- a/crates/markkit/src/spreadsheet.rs +++ b/crates/markkit/src/spreadsheet.rs @@ -34,6 +34,7 @@ use crate::render::escape_table_cell; pub(crate) struct SheetOutput { pub doc: Document, pub format: Format, + pub title: Option<String>, pub warnings: Vec<String>, pub sheets: usize, pub rows: usize, @@ -63,7 +64,8 @@ enum Container { } pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput, Error> { - let container = if bytes.starts_with(b"PK") { + let mut patched = None; + let (container, title) = if bytes.starts_with(b"PK") { let mut zip = ZipGuard::open(bytes, config).map_err(archive_err)?; let has = |zip: &ZipGuard<'_>, name: &str| zip.names().any(|n| n.eq_ignore_ascii_case(name)); @@ -79,20 +81,29 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput return Err(err("ZIP container is not an XLSX or ODS workbook")); }; zip.validate_all().map_err(archive_err)?; - if let Container::Ods = container { + let title = if let Container::Ods = container { let content = zip .read("content.xml") .map_err(archive_err)? .ok_or_else(|| err("ODS is missing content.xml"))?; prescan_ods(&content, config.max_spreadsheet_cells)?; - } - container + read_dc_title(&mut zip, "meta.xml")? + } else { + if !has(&zip, "_rels/.rels") { + patched = Some(with_package_rels(&mut zip)?); + } + read_dc_title(&mut zip, "docProps/core.xml")? + }; + (container, title) } else if bytes.starts_with(OLE2_MAGIC) { - prescan_xls(bytes, config)?; - Container::Xls + let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)) + .map_err(|e| err(format!("invalid XLS container: {e}")))?; + prescan_xls(&mut comp, config)?; + (Container::Xls, xls_title(&mut comp)) } else { return Err(err("not a spreadsheet (expected XLSX, XLS or ODS)")); }; + let bytes = patched.as_deref().unwrap_or(bytes); let mut out = Collector::new(config); match container { @@ -116,19 +127,19 @@ pub(crate) fn parse(bytes: &[u8], config: &MarkkitBuilder) -> Result<SheetOutput Err(e) => out.warn(format!("sheet_error: sheet '{name}': {e}")), } } - out.finish(Format::Xlsx) + out.finish(Format::Xlsx, title) } Container::Ods => { let mut wb: Ods<_> = Ods::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Ods) + out.finish(Format::Ods, title) } Container::Xls => { let mut wb: Xls<_> = Xls::new(Cursor::new(bytes)).map_err(err)?; let sel = out.select(wb.sheets_metadata()); read_ranges(&mut wb, sel, &mut out, config); - out.finish(Format::Xls) + out.finish(Format::Xls, title) } } } @@ -323,11 +334,20 @@ impl Grid { } row }); - let header = rows.next()?; + let first = rows.next()?; + let rest: Vec<Vec<String>> = rows.collect(); + // GFM tables need a header row. A sheet with a single row is data, not a + // header: label its columns A, B, … so the row renders as a data row. + let (header, rows) = if rest.is_empty() { + let letters = (min_col..min_col + width as u32).map(col_name).collect(); + (letters, vec![first]) + } else { + (first, rest) + }; Some(SheetTable { range, header, - rows: rows.collect(), + rows, truncated_rows: self.truncated_rows, truncated_cols: self.truncated_cols, }) @@ -427,10 +447,11 @@ impl<'c> Collector<'c> { }); } - fn finish(self, format: Format) -> Result<SheetOutput, Error> { + fn finish(self, format: Format, title: Option<String>) -> Result<SheetOutput, Error> { Ok(SheetOutput { doc: Document::new(self.nodes), format, + title, warnings: self.warnings, sheets: self.sheets, rows: self.rows, @@ -438,6 +459,122 @@ impl<'c> Collector<'c> { } } +// ─── Package fixes and document title ─────────────────────────────── + +/// Longest title kept (characters); longer titles are cut. +const MAX_TITLE_CHARS: usize = 1024; + +/// calamine locates the workbook through `_rels/.rels` and fails without it. +/// Rebuild the (already validated) archive with the standard relationship to +/// `xl/workbook.xml`; entries are copied raw, without recompression. +fn with_package_rels(zip: &mut ZipGuard<'_>) -> Result<Vec<u8>, Error> { + const RELS: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#; + let zerr = |e: zip::result::ZipError| err(format!("cannot rebuild XLSX package: {e}")); + let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); + for i in 0..zip.archive.len() { + let entry = zip.archive.by_index_raw(i).map_err(zerr)?; + w.raw_copy_file(entry).map_err(zerr)?; + } + let opts = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + w.start_file("_rels/.rels", opts).map_err(zerr)?; + std::io::Write::write_all(&mut w, RELS.as_bytes()) + .map_err(|e| err(format!("cannot rebuild XLSX package: {e}")))?; + Ok(w.finish().map_err(zerr)?.into_inner()) +} + +fn clean_title(raw: &str) -> Option<String> { + let t: String = raw + .split_whitespace() + .collect::<Vec<_>>() + .join(" ") + .chars() + .take(MAX_TITLE_CHARS) + .collect(); + (!t.is_empty()).then_some(t) +} + +/// `dc:title` from OOXML `docProps/core.xml` or ODF `meta.xml`. Best effort: +/// a missing or malformed part yields `None`; only archive limits are errors. +fn read_dc_title(zip: &mut ZipGuard<'_>, part: &str) -> Result<Option<String>, Error> { + let Some(xml) = zip.read(part).map_err(archive_err)? else { + return Ok(None); + }; + Ok(dc_title(&xml)) +} + +fn dc_title(xml: &[u8]) -> Option<String> { + let root = crate::xml::parse(xml).ok()?; + clean_title(&root.find("title")?.text()) +} + +/// Title (PIDSI_TITLE) from the XLS `\u{5}SummaryInformation` property set +/// ([MS-OLEPS]). Best effort: anything malformed yields `None`. +fn xls_title<F: Read + Seek>(comp: &mut cfb::CompoundFile<F>) -> Option<String> { + const CAP: u64 = 1024 * 1024; + let mut data = Vec::new(); + comp.open_stream("/\u{5}SummaryInformation") + .ok()? + .take(CAP) + .read_to_end(&mut data) + .ok()?; + summary_title(&data) +} + +/// Decode UTF-16LE (lossy; a trailing odd byte is ignored). +fn utf16le(bytes: &[u8]) -> String { + let (pairs, _) = bytes.as_chunks::<2>(); + let units: Vec<u16> = pairs.iter().map(|b| u16::from_le_bytes(*b)).collect(); + String::from_utf16_lossy(&units) +} + +fn summary_title(d: &[u8]) -> Option<String> { + const PID_CODEPAGE: u32 = 1; + const PID_TITLE: u32 = 2; + const VT_I2: u32 = 0x0002; + const VT_LPSTR: u32 = 0x001E; + const VT_LPWSTR: u32 = 0x001F; + if read_u16(d, 0)? != 0xFFFE || read_u32(d, 24)? == 0 { + return None; + } + let set = read_u32(d, 44)? as usize; + let count = read_u32(d, set.checked_add(4)?)? as usize; + let mut codepage = None; + let mut title_at = None; + for i in 0..count.min(1024) { + let entry = set.checked_add(8 + i * 8)?; + let (pid, off) = (read_u32(d, entry)?, read_u32(d, entry + 4)? as usize); + let at = set.checked_add(off)?; + match pid { + PID_CODEPAGE if read_u32(d, at)? & 0xFFFF == VT_I2 => { + codepage = Some(read_u16(d, at + 4)?); + } + PID_TITLE => title_at = Some(at), + _ => {} + } + } + let at = title_at?; + let typ = read_u32(d, at)? & 0xFFFF; + let n = read_u32(d, at + 4)? as usize; + let start = at + 8; + let raw = match typ { + VT_LPWSTR => utf16le(d.get(start..start.checked_add(n.checked_mul(2)?)?)?), + VT_LPSTR => { + let bytes = d.get(start..start.checked_add(n)?)?; + match codepage { + Some(1200) => utf16le(bytes), + // UTF-8 when valid, otherwise Latin-1 (close to Windows-1252). + _ => match std::str::from_utf8(bytes) { + Ok(s) => s.to_string(), + Err(_) => bytes.iter().map(|&b| char::from(b)).collect(), + }, + } + } + _ => return None, + }; + clean_title(raw.trim_end_matches('\0')) +} + // ─── XLS pre-scan (BIFF records) ──────────────────────────────────── fn read_u16(d: &[u8], at: usize) -> Option<u16> { @@ -461,9 +598,10 @@ fn records(stream: &[u8]) -> impl Iterator<Item = (u16, &[u8])> { }) } -fn prescan_xls(bytes: &[u8], config: &MarkkitBuilder) -> Result<(), Error> { - let mut comp = cfb::CompoundFile::open(Cursor::new(bytes)) - .map_err(|e| err(format!("invalid XLS container: {e}")))?; +fn prescan_xls<F: Read + Seek>( + comp: &mut cfb::CompoundFile<F>, + config: &MarkkitBuilder, +) -> Result<(), Error> { let name = ["/Workbook", "/Book", "/WORKBOOK", "/BOOK"] .into_iter() .find(|n| comp.is_stream(n)) @@ -737,6 +875,92 @@ mod tests { assert!(t.truncated_rows); } + #[test] + fn column_names() { + assert_eq!(col_name(0), "A"); + assert_eq!(col_name(25), "Z"); + assert_eq!(col_name(26), "AA"); + assert_eq!(col_name(16_383), "XFD"); + assert_eq!(col_name(701), "ZZ"); + assert_eq!(col_name(702), "AAA"); + assert!(!col_name(u32::MAX).is_empty()); + } + + #[test] + fn single_row_becomes_data_under_column_letters() { + let cfg = MarkkitBuilder::default(); + let mut g = Grid::new(&cfg); + g.add(4, 2, "x".into()); + g.add(4, 3, "y".into()); + let t = g.into_table().unwrap(); + assert_eq!(t.header, vec!["C", "D"]); + assert_eq!(t.rows, vec![vec!["x".to_string(), "y".to_string()]]); + } + + #[test] + fn dc_title_reads_and_normalizes() { + let core = br#"<cp:coreProperties xmlns:cp="c" xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:title> Q3 + Budget & Plan </dc:title></cp:coreProperties>"#; + assert_eq!(dc_title(core).as_deref(), Some("Q3 Budget & Plan")); + assert_eq!(dc_title(b"<a><dc:title> </dc:title></a>"), None); + assert_eq!(dc_title(b"<a></a>"), None); + assert_eq!(dc_title(b"\xff\xfe<<"), None); + let long = format!("<title>{}", "x".repeat(5000)); + assert_eq!(dc_title(long.as_bytes()).unwrap().len(), MAX_TITLE_CHARS); + } + + fn summary_info(codepage: u16, typ: u32, value: &[u8], count: u32) -> Vec { + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + // Property set: size, count, 2 × (pid, offset), codepage, title. + let mut set = Vec::new(); + set.extend_from_slice(&0u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&1u32.to_le_bytes()); + set.extend_from_slice(&24u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&32u32.to_le_bytes()); + set.extend_from_slice(&2u32.to_le_bytes()); + set.extend_from_slice(&u32::from(codepage).to_le_bytes()); + set.extend_from_slice(&typ.to_le_bytes()); + set.extend_from_slice(&count.to_le_bytes()); + set.extend_from_slice(value); + d.extend_from_slice(&set); + d + } + + #[test] + fn summary_title_decodes_codepages() { + let d = summary_info(1252, 0x1E, b"Sales\0", 6); + assert_eq!(summary_title(&d).as_deref(), Some("Sales")); + let d = summary_info(1252, 0x1E, b"Caf\xe9\0", 5); + assert_eq!(summary_title(&d).as_deref(), Some("Café")); + let d = summary_info(65001, 0x1E, "Звіт\0".as_bytes(), 9); + assert_eq!(summary_title(&d).as_deref(), Some("Звіт")); + let wide: Vec = "Hi\0".encode_utf16().flat_map(u16::to_le_bytes).collect(); + let d = summary_info(1200, 0x1F, &wide, 3); + assert_eq!(summary_title(&d).as_deref(), Some("Hi")); + } + + #[test] + fn summary_title_rejects_malformed() { + assert_eq!(summary_title(b""), None); + assert_eq!(summary_title(&[0xFE, 0xFF, 0, 0]), None); + // Length pointing past the stream. + let d = summary_info(1252, 0x1E, b"abc", u32::MAX); + assert_eq!(summary_title(&d), None); + // Property offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[48 + 20..48 + 24].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + // Section offset pointing past the stream. + let mut d = summary_info(1252, 0x1E, b"abc\0", 4); + d[44..48].copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(summary_title(&d), None); + } + #[test] fn biff_scan_rejects_inverted_mulrk() { let mut s = Vec::new(); diff --git a/crates/markkit/tests/common/mod.rs b/crates/markkit/tests/common/mod.rs index 47bbd2b..268d897 100644 --- a/crates/markkit/tests/common/mod.rs +++ b/crates/markkit/tests/common/mod.rs @@ -416,15 +416,70 @@ pub fn biff8_workbook(sheets: &[XlsSheet]) -> Vec { /// Wrap a workbook stream into an OLE2 compound file. pub fn xls_from_stream(stream: &[u8]) -> Vec { + ole2(&[("/Workbook", stream)]) +} + +/// Build an OLE2 compound file from `(path, data)` streams. +pub fn ole2(streams: &[(&str, &[u8])]) -> Vec { let mut comp = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); - { - let mut s = comp.create_stream("/Workbook").unwrap(); - s.write_all(stream).unwrap(); + for (path, data) in streams { + let mut s = comp.create_stream(path).unwrap(); + s.write_all(data).unwrap(); } comp.flush().unwrap(); comp.into_inner().into_inner() } +/// `\u{5}SummaryInformation` property set with a code page and a VT_LPSTR title. +pub fn summary_information(codepage: u16, title: &[u8]) -> Vec { + let mut value = title.to_vec(); + value.push(0); + let mut set = Vec::new(); + for v in [ + 0u32, + 2, + 1, + 24, + 2, + 32, + 2, + u32::from(codepage), + 0x1E, + value.len() as u32, + ] { + set.extend_from_slice(&v.to_le_bytes()); + } + set.extend_from_slice(&value); + let mut d = vec![0u8; 48]; + d[0..2].copy_from_slice(&0xFFFEu16.to_le_bytes()); + d[24..28].copy_from_slice(&1u32.to_le_bytes()); + d[44..48].copy_from_slice(&48u32.to_le_bytes()); + d.extend_from_slice(&set); + d +} + +/// Rebuild a ZIP without the `drop` entries and with `add` appended. +pub fn rezip(bytes: &[u8], drop: &[&str], add: &[(&str, &[u8])]) -> Vec { + let mut zip = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + let mut entries = Vec::new(); + for i in 0..zip.len() { + let mut f = zip.by_index(i).unwrap(); + let name = f.name().unwrap().to_string(); + if drop.contains(&name.as_str()) { + continue; + } + let mut data = Vec::new(); + std::io::Read::read_to_end(&mut f, &mut data).unwrap(); + entries.push((name, data)); + } + let mut refs: Vec<(&str, &[u8])> = entries + .iter() + .map(|(n, d)| (n.as_str(), d.as_slice())) + .collect(); + refs.extend_from_slice(add); + zip_bytes(&refs) +} + pub fn xls(sheets: &[XlsSheet]) -> Vec { xls_from_stream(&biff8_workbook(sheets)) } diff --git a/crates/markkit/tests/document_security.rs b/crates/markkit/tests/document_security.rs index 693ea25..ce64334 100644 --- a/crates/markkit/tests/document_security.rs +++ b/crates/markkit/tests/document_security.rs @@ -319,6 +319,24 @@ fn valid_fixtures() -> Vec> { dims: Some((0, 1, 0, 2)), cells: vec![XlsCell::Label(0, 0, "a"), XlsCell::Num(0, 1, 2.0)], }])); + // Title sources: XLS SummaryInformation, XLSX core.xml + no `_rels/.rels`. + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "a")], + }]); + let summary = summary_information(1252, b"Title"); + v.push(ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ])); + let core = b"T"; + let book = xlsx_with( + &[Sheet::new("S", vec![vec![Cell::S("a")]])], + &[], + &[("docProps/core.xml", core)], + ); + v.push(rezip(&book, &["_rels/.rels"], &[])); } v } @@ -379,7 +397,7 @@ proptest! { /// Byte flips in valid documents never panic. #[test] fn mutated_fixtures_never_panic( - idx in 0usize..4, + idx in 0usize..6, flips in proptest::collection::vec((any::(), any::()), 1..16), ) { let fixtures = valid_fixtures(); diff --git a/crates/markkit/tests/spreadsheet_integration.rs b/crates/markkit/tests/spreadsheet_integration.rs index 3c54e02..bc9eb4b 100644 --- a/crates/markkit/tests/spreadsheet_integration.rs +++ b/crates/markkit/tests/spreadsheet_integration.rs @@ -50,9 +50,10 @@ fn xlsx_multiple_sheets_and_empty_sheet_skipped() { let e = extract(&bytes); assert_eq!( e.markdown, - "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| x | y |\n| --- | --- |\n" + "## A\n\n| h |\n| --- |\n| 1 |\n\n## B\n\n| A | B |\n| --- | --- |\n| x | y |\n" ); assert_eq!(e.stats.sheets, Some(2)); + assert_eq!(e.stats.rows, Some(2)); } #[test] @@ -276,6 +277,121 @@ fn xls_by_content_type() { assert!(e.markdown.contains("| x |"), "{}", e.markdown); } +// ─── Single-row sheets, titles, package quirks ────────────────────── + +#[test] +fn single_row_sheet_renders_as_data_under_column_letters() { + // One row starting at column C: header is the real column letters. + let data = r#"only7"#; + let bytes = xlsx_with( + &[Sheet::new("One", vec![])], + &[Some(worksheet_xml(data))], + &[], + ); + let e = extract(&bytes); + assert_eq!( + e.markdown, + "## One\n\n| C | D |\n| --- | --- |\n| only | 7 |\n" + ); + assert_eq!(e.stats.rows, Some(1)); + + // Two rows: the first is still the header. + let bytes = xlsx(&[Sheet::new("Two", vec![vec![S("h")], vec![S("v")]])]); + assert_eq!( + extract(&bytes).markdown, + "## Two\n\n| h |\n| --- |\n| v |\n" + ); +} + +#[test] +fn xlsx_title_from_core_properties() { + let core = br#"Quarterly & Annual"#; + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[("docProps/core.xml", core)], + ); + assert_eq!(extract(&bytes).title.as_deref(), Some("Quarterly & Annual")); + // No core properties: no title. + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("v")]])]); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn xlsx_malformed_core_properties_do_not_fail_extraction() { + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])], + &[], + &[( + "docProps/core.xml", + b"x".repeat(10_000), "t"); + let bytes = xlsx_with( + &[Sheet::new("S", vec![vec![S("v")]])], + &[], + &[("docProps/core.xml", deep.as_bytes())], + ); + assert_eq!(extract(&bytes).title, None); +} + +#[test] +fn ods_title_from_meta() { + let content = format!( + r#"{}"#, + ods_string_cell("v") + ); + let meta = br#"ODS Report"#; + let bytes = rezip(&ods(&content), &[], &[("meta.xml", meta)]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Ods); + assert_eq!(e.title.as_deref(), Some("ODS Report")); +} + +#[test] +fn xls_title_from_summary_information() { + let stream = biff8_workbook(&[XlsSheet { + name: "S", + dims: None, + cells: vec![XlsCell::Label(0, 0, "h"), XlsCell::Label(1, 0, "v")], + }]); + let summary = summary_information(1252, b"Legacy Budget"); + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", &summary), + ]); + let e = extract(&bytes); + assert_eq!(e.format, Format::Xls); + assert_eq!(e.title.as_deref(), Some("Legacy Budget")); + + // Garbage property set: no title, extraction still succeeds. + let bytes = ole2(&[ + ("/Workbook", &stream), + ("/\u{5}SummaryInformation", b"\xFE\xFFjunk"), + ]); + let e = extract(&bytes); + assert_eq!(e.title, None); + assert!(e.markdown.contains("| v |")); +} + +#[test] +fn xlsx_without_package_relationships_still_parses() { + let bytes = xlsx(&[Sheet::new("S", vec![vec![S("h")], vec![S("v")]])]); + let bytes = rezip(&bytes, &["_rels/.rels"], &[]); + let e = extract(&bytes); + assert_eq!(e.markdown, "## S\n\n| h |\n| --- |\n| v |\n"); +} + +#[test] +fn xlsb_content_type_is_not_xlsx() { + let ct = "application/vnd.ms-excel.sheet.binary.macroEnabled.12"; + assert_eq!(Format::from_content_type(ct), Format::Xlsb); + assert_eq!(Format::from_file_name("book.xlsb"), Format::Xlsb); +} + // ─── Provenance ────────────────────────────────────────────────────── #[test] diff --git a/knowledge/formats/format-support.md b/knowledge/formats/format-support.md index f1f915b..0ae6deb 100644 --- a/knowledge/formats/format-support.md +++ b/knowledge/formats/format-support.md @@ -44,7 +44,7 @@ impl Markkit { pub struct Extraction { // #[non_exhaustive] pub markdown: String, - pub title: Option, // HTML og:title / JSON-LD / ; DOCX core.xml dc:title, else Title style + pub title: Option<String>, // see "Document titles" below pub format: Format, pub warnings: Vec<String>, // "<code>: <message>" pub stats: ExtractionStats, // pages / sheets / rows / paragraphs / tables (Option each) @@ -52,6 +52,22 @@ pub struct Extraction { // #[non_exhaustive] } ``` +Non-empty `markdown` always ends with exactly one `\n` (POSIX text convention; outputs +concatenate cleanly); an empty document renders `""`. This is intended and stable. + +### Document titles + +| Format | Source | +|---|---| +| HTML | `og:title` / JSON-LD / `<title>` (see [HTML Extraction](html-extraction.md)) | +| DOCX | `docProps/core.xml` `dc:title` (via `_rels/.rels`), else the first `Title`-style paragraph | +| XLSX | `docProps/core.xml` `dc:title` | +| ODS | `meta.xml` `dc:title` | +| XLS | `\u{5}SummaryInformation` property set, `PIDSI_TITLE` (code page aware: 1200 UTF-16, UTF-8, else Latin-1) | + +Whitespace is collapsed, titles are capped at 1024 characters, and a missing or +malformed property part yields `None` without failing the extraction. + `extract_bytes(bytes, content_type)` delegates to `extract_document_sync` and returns only the Markdown. Parsing is CPU-bound; async callers should use the sync variant in `spawn_blocking`. @@ -61,12 +77,14 @@ Warning codes: `truncated_sheet`, `hidden_sheet_skipped`, `sheet_error`. ## Format Detection Priority order (`Format::detect`): -1. Content type (`DocumentHint::content_type`; parameters ignored). Generic types such as +1. Content type (`DocumentHint::content_type`; parameters ignored). XLSB + (`application/vnd.ms-excel.sheet.binary.macroEnabled.12`) is matched before the + generic `ms-excel.sheet` → XLSX rule. Generic types such as `application/octet-stream` map to `Unknown` and fall through. 2. File extension (`DocumentHint::file_name`; URL query/fragment ignored). 3. Magic bytes (`Format::sniff`): - `PK\x03\x04` ZIP: ODF `mimetype` entry → ODS; `word/document*` → DOCX; - `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored + `xl/workbook.bin` → XLSB; `xl/workbook.*` → XLSX; `ppt/presentation.*` → PPTX (part names are stored uncompressed, so no archive parsing is needed for detection). - OLE2 (`D0 CF 11 E0`): UTF-16 `WordDocument` stream → DOC; `Workbook`/`Book` → XLS. - `%PDF-` within the first 1 KiB → PDF. @@ -96,7 +114,11 @@ numbering through the main part's relationships. - One `## <Sheet name>` section per non-empty worksheet, in workbook order, followed by a GFM table. The first non-empty row is the header; fully empty rows and leading empty - columns are dropped. + columns are dropped. A sheet with a single non-empty row has no header: its columns + are labelled with their Excel letters (`| C | D |`) and the row renders as data + (counted in `stats.rows`). +- XLSX packages without `_rels/.rels` (hand-built files) get the standard + officeDocument relationship to `xl/workbook.xml` before calamine opens them. - Cells: numbers without trailing `.0` for integers; booleans `true`/`false`; dates as `YYYY-MM-DD[ HH:MM:SS]`, durations `H:MM:SS`; errors as Excel shows them (`#DIV/0!`). Pipes are escaped (`\|`), line breaks flattened to spaces. diff --git a/knowledge/log.md b/knowledge/log.md index 63963b1..ffa79fe 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Markkit Knowledge Update Log +## 2026-10-27 + +* **Spreadsheet follow-ups (#11)**: single-row sheets render as data under column letters; XLSX/ODS/XLS titles come from document properties; XLSB content type no longer maps to XLSX; XLSX without `_rels/.rels` parses; trailing-newline output documented as intended; release profile switched to `panic = "unwind"` so `catch_unwind` is effective in markkit's own binaries. See [Format Support](formats/format-support.md) and [Threat Model](security/threat-model.md) T6c. + ## 2026-10-26 * **Relative URLs**: links, images and metadata URLs resolve against `base_url` (or the final `extract_url` response URL) plus `<base href>`. Only scheme-less references change; page bases are limited to `http(s)`. New `extract_url_document`. See [HTML Extraction](formats/html-extraction.md) and [Threat Model](security/threat-model.md) T1. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index 6e322d2..47c0ea1 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -208,7 +208,11 @@ process, or smuggling content past a human reviewer. DIMENSIONS, inverted MULRK column ranges, out-of-range sheet offsets). - XML: depth capped at 256, tree built iteratively, unclosed elements tolerated. - Parsers run under `catch_unwind` (defence in depth; only effective with - `panic = "unwind"` -- the workspace release profile uses `abort`). + `panic = "unwind"`, which the workspace release profile uses so the CLI and the + Python/Node bindings survive a parser panic; embedders building with `abort` lose + this layer). +- XLS `SummaryInformation` title parsing is bounds-checked (1 MiB stream cap, checked + offsets, 1024-property cap); failures yield no title, never an error. #### T6d: XML entity attacks (billion laughs, XXE) - quick-xml never expands DTD-declared or external entities; such references are dropped. @@ -234,6 +238,8 @@ process, or smuggling content past a human reviewer. `spreadsheet_integration::xlsx_sparse_far_cell_does_not_allocate_grid` - `docx_billion_laughs_not_expanded`, `docx_external_entity_not_resolved`, `docx_deep_nesting_does_not_overflow_stack` +- `spreadsheet::tests::summary_title_rejects_malformed`, + `spreadsheet_integration::xlsx_malformed_core_properties_do_not_fail_extraction` - `truncated_fixtures_never_panic`, `random_bytes_never_panic`, `random_container_bytes_never_panic`, `mutated_fixtures_never_panic` - `spreadsheet_integration::xlsx_hidden_sheet_skipped_by_default`,