`s took
+ 12.6 s and 292 KiB of unclosed nested tables ran past the 60 s timeout; tags nested past
+ `max_nesting_depth` are now dropped (text kept) before html5ever parses, so both take
+ milliseconds.
+- HTML: rows of nested tables are no longer copied into every enclosing table (output grew 4x
+ per nesting level).
+- HTML: text flattened past `max_nesting_depth` no longer includes `
"
+ );
+ assert!(matches!(cap_nesting(&html, 5, true), Cow::Borrowed(_)));
+ }
+
+ #[test]
+ fn optional_end_tags_and_voids_do_not_count() {
+ let html = "
bcd".repeat(1000);
+ assert!(matches!(cap_nesting(&html, 5, true), Cow::Borrowed(_)));
+ }
+
+ #[test]
+ fn implied_close_by_ancestor_end_tag() {
+ // `` closes the unclosed spans inside it; depth must not drift.
+ let html = "x
".repeat(1000);
+ assert!(matches!(cap_nesting(&html, 10, true), Cow::Borrowed(_)));
+ }
+
+ #[test]
+ fn raw_text_end_tags_survive_while_dropping() {
+ // Dropping `` would turn the rest of the page into script.
+ let html = format!(
+ "{}after
{}",
+ "".repeat(50),
+ "
".repeat(50)
+ );
+ let capped = cap_nesting(&html, 10, true);
+ assert!(capped.contains(""));
+ assert!(capped.contains("after"));
+ }
+
+ #[test]
+ fn uncounted_end_tags_do_not_consume_overflow() {
+ let html = format!(
+ "{}a
b {}",
+ "".repeat(20),
+ "
".repeat(20)
+ );
+ let capped = cap_nesting(&html, 10, true);
+ // Past the cap the p/li tags go too, their text stays, and their end
+ // tags do not close dropped divs.
+ assert!(capped.contains("ab"));
+ assert_eq!(capped.matches("").count(), 10);
+ assert_eq!(capped.matches("
").count(), 10);
+ }
+
+ #[test]
+ fn non_content_past_the_cap_is_dropped_whole() {
+ let html = format!(
+ "{}tplin after
",
+ "".repeat(20)
+ );
+ let capped = cap_nesting(&html, 10, true);
+ assert!(!capped.contains("tpl") && !capped.contains("in<") && !capped.contains("f"));
+ assert!(capped.contains("after"));
+ let objects = "
".repeat(100_000) + "x";
+ assert!(!cap_nesting(&objects, 10, true).contains('x'));
+ }
+
+ #[test]
+ fn hidden_tags_past_the_cap_are_dropped_whole() {
+ // 11 divs: the last is dropped, so even optional-end tags (`p`) are
+ // in the dropped region (at the cap they would stay in the DOM, where
+ // the walker's flattener checks them).
+ let deep = "".repeat(11);
+ for tag in [
+ r#"
S "#,
+ r#"
S
"#,
+ r#"
S
"#,
+ r#"
S "#,
+ ] {
+ let html = format!("{deep}a{tag}b");
+ let html = html.as_str();
+ let capped = cap_nesting(html, 10, true);
+ assert!(
+ capped.contains("ab") && !capped.contains('S'),
+ "{tag}: {capped}"
+ );
+ // Opted in: only the tag goes, the text stays.
+ assert!(cap_nesting(html, 10, false).contains('S'), "{tag}");
+ }
+ let shown = format!("{deep}
S T ");
+ let capped = cap_nesting(&shown, 10, true);
+ assert!(capped.contains('S') && capped.contains('T'), "{capped}");
+ }
+
+ #[test]
+ fn raw_attrs_parse_quotes_and_bare_values() {
+ assert_eq!(
+ raw_attrs(r#" a b=1 c = "x y" d='z>' /"#),
+ [("a", ""), ("b", "1"), ("c", "x y"), ("d", "z>")]
+ );
+ assert_eq!(raw_attrs(r#" e="unterminated"#), [("e", "unterminated")]);
+ }
+
+ #[test]
+ fn malformed_tails_do_not_panic() {
+ for s in [
+ "<", "", "
(attr: impl Fn(&str) -> Option<&'v str>) -> bool {
+ if attr("hidden").is_some_and(|v| !v.eq_ignore_ascii_case("until-found")) {
return true;
}
- if el
- .attr("aria-hidden")
- .is_some_and(|v| v.trim().eq_ignore_ascii_case("true"))
- {
+ if attr("aria-hidden").is_some_and(|v| v.trim().eq_ignore_ascii_case("true")) {
return true;
}
- let Some(style) = el.attr("style") else {
+ let Some(style) = attr("style") else {
return false;
};
style.split(';').any(|decl| {
@@ -173,12 +179,21 @@ impl<'a> HtmlParser<'a> {
/// Parse an HTML string into a [`Document`].
pub fn parse(&self, html: &str) -> Result
{
- self.parse_dom(&Html::parse_document(html))
+ self.parse_dom(&self.parse_html5ever(html))
+ }
+
+ /// html5ever parse of `html` with nesting capped first (see `nesting`).
+ fn parse_html5ever(&self, html: &str) -> Html {
+ Html::parse_document(&nesting::cap_nesting(
+ html,
+ self.max_depth + NESTING_SLACK,
+ !self.config.include_hidden_content,
+ ))
}
/// Parse an HTML string into a [`Document`] plus page title and metadata.
pub(crate) fn parse_page(&self, html: &str) -> Result<(Document, PageInfo), Error> {
- let dom = Html::parse_document(html);
+ let dom = self.parse_html5ever(html);
let this = self.for_document(&dom);
Ok((
this.convert_dom(&dom)?,
@@ -325,7 +340,7 @@ impl<'a> HtmlParser<'a> {
}
if depth >= self.max_depth {
// At max depth, just extract text
- let text: String = parent.text().collect();
+ let text = flat_text(parent, !self.config.include_hidden_content);
let trimmed = text.trim();
if !trimmed.is_empty() {
out.push(Node::Text(trimmed.to_string()));
@@ -410,7 +425,7 @@ impl<'a> HtmlParser<'a> {
return None;
}
if depth >= self.max_depth {
- let text: String = el.text().collect();
+ let text = flat_text(el, !self.config.include_hidden_content);
let trimmed = text.trim();
if trimmed.is_empty() {
return None;
@@ -609,9 +624,7 @@ impl<'a> HtmlParser<'a> {
"hr" => Some(Node::ThematicBreak),
// Skip non-content elements
- "script" | "style" | "noscript" | "template" | "iframe" | "object" | "embed"
- | "form" | "input" | "button" | "select" | "textarea" | "svg" | "canvas" | "video"
- | "audio" | "source" | "track" | "map" | "area" => None,
+ name if NON_CONTENT.contains(&name) => None,
// Unknown element: try to extract inline content
_ => {
@@ -927,7 +940,7 @@ impl<'a> HtmlParser<'a> {
/// `` that is a footnote reference: record target and build the node.
fn footnote_reference(&self, el: scraper::ElementRef<'_>) -> Option {
let target = el.value().attr("href")?.strip_prefix('#')?;
- let text: String = el.text().collect();
+ let text = flat_text(el, !self.config.include_hidden_content);
let text = text
.trim()
.trim_start_matches('[')
@@ -951,7 +964,7 @@ impl<'a> HtmlParser<'a> {
return nodes;
}
if depth >= self.max_depth {
- let text: String = el.text().collect();
+ let text = flat_text(el, !self.config.include_hidden_content);
let trimmed = text.trim();
if !trimmed.is_empty() {
nodes.push(Node::Text(trimmed.to_string()));
@@ -1040,17 +1053,15 @@ impl<'a> HtmlParser<'a> {
exclude: &[Selector],
depth: usize,
) -> Option {
- let thead_sel = Selector::parse("thead").unwrap();
- let tbody_sel = Selector::parse("tbody").unwrap();
- let tr_sel = Selector::parse("tr").unwrap();
-
let mut headers: Vec = Vec::new();
let mut alignments: Vec = Vec::new();
let mut rows: Vec> = Vec::new();
- if let Some(thead) = table_el.select(&thead_sel).next()
- && let Some(tr) = thead.select(&tr_sel).next()
- {
+ // Direct structure only: a descendant query would also pick up the
+ // rows of nested tables, duplicating them into every enclosing table
+ // (output and work grew 4x per nesting level). THREAT[T2c].
+ let (head_row, body_rows) = table_rows(table_el);
+ if let Some(tr) = head_row {
let ths = row_cells(tr, &["th", "td"]);
alignments = column_alignments(&ths);
headers = ths
@@ -1059,13 +1070,6 @@ impl<'a> HtmlParser<'a> {
.collect();
}
- let body_rows: Vec> =
- if let Some(tbody) = table_el.select(&tbody_sel).next() {
- tbody.select(&tr_sel).collect()
- } else {
- table_el.select(&tr_sel).collect()
- };
-
for tr in body_rows {
if self.is_excluded(&tr, exclude) {
continue;
@@ -1135,6 +1139,64 @@ impl<'a> HtmlParser<'a> {
}
}
+/// Text of a subtree flattened at `max_nesting_depth`, without the content of
+/// [`NON_CONTENT`] elements (`el.text()` leaked `tpl after
{}",
+ "".repeat(depth),
+ "
".repeat(depth)
+ );
+ let md = convert(&html);
+ assert!(md.contains("ok") && md.contains("after"), "{depth}: {md}");
+ for bad in [
+ "SECRET_JS",
+ "color:red",
+ "tpl",
+ "