Skip redundant regex work when scoring search passages

src/docpull/pack_tools.py

@@ -8,7 +8,7 @@
 import re
 from collections import Counter
 from dataclasses import dataclass, field
-from functools import partial
+from functools import lru_cache, partial
 from pathlib import Path
 from typing import Any
 from urllib.parse import urlparse
@@ -57,6 +57,8 @@
 
 _WORD_RE = re.compile(r"[a-z0-9][a-z0-9-]{2,}", re.IGNORECASE)
 _MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)]\([^)]+\)")
+_LIST_MARKER_RE = re.compile(r"^\s*[-*+]\s+", re.MULTILINE)
+_HEADING_MARKER_RE = re.compile(r"^\s{0,3}#{1,6}\s*", re.MULTILINE)
 _EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")
 _MONEY_RE = re.compile(
     r"(?<!\w)(?:\$|USD\s*)\d[\d,]*(?:\.\d+)?(?:\s?(?:k|K|m|M|b|B|million|billion|thousand))?\b"
@@ -2533,10 +2535,15 @@ def _search_score(
     return score, matched_terms
 
 
+@lru_cache(maxsize=1024)
+def _term_pattern(term: str) -> re.Pattern[str]:
+    return re.compile(rf"\b{re.escape(term)}\b", re.IGNORECASE)
+
+
 def _term_count(text: str, term: str) -> int:
     if not text or not term:
         return 0
-    return len(re.findall(rf"\b{re.escape(term)}\b", text, flags=re.IGNORECASE))
+    return len(_term_pattern(term).findall(text))
 
 
 def _best_search_excerpt(content: str, query_terms: list[str], phrase: str) -> str:
@@ -2603,12 +2610,14 @@ def _truncate_text(value: str, max_chars: int) -> str:
 
 
 def _clean_passage(value: str) -> str:
-    text = _MARKDOWN_LINK_RE.sub(r"\1", value)
-    text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE)
-    text = re.sub(r"^\s{0,3}#{1,6}\s*", "", text, flags=re.MULTILINE)
-    text = text.replace("`", "")
-    text = re.sub(r"\s+", " ", text)
-    return text.strip()
+    # Skip each pattern when its required character is absent; most passages have no markdown.
+    text = _MARKDOWN_LINK_RE.sub(r"\1", value) if "](" in value else value
+    if "-" in text or "*" in text or "+" in text:
+        text = _LIST_MARKER_RE.sub("", text)
+    if "#" in text:
+        text = _HEADING_MARKER_RE.sub("", text)
+    # str.split() and regex \s use the same Unicode whitespace set.
+    return " ".join(text.replace("`", "").split())
 
 
 def _nearest_sentence(content: str, start: int, end: int) -> str:

tests/test_pack_tools.py

@@ -805,3 +805,37 @@ def test_pack_diff_cli_writes_outputs(tmp_path: Path) -> None:
     semantic = json.loads((new_pack / "semantic.diff.json").read_text(encoding="utf-8"))
     assert semantic["summary"]["removed_section"] == 1
     assert "Added" in (tmp_path / "diff.md").read_text(encoding="utf-8")
+
+
+def test_search_scoring_matches_pre_fast_path_values() -> None:
+    # Expected values were recorded from the uncompiled-regex implementation.
+    content = (
+        "- Retry  with [backoff](https://b.example) and `retry()`.\n\n"
+        "### Retry backoff\nRETRY retrying retry_count"
+    )
+    assert pack_tools_module._clean_passage(content) == (
+        "Retry with backoff and retry(). Retry backoff RETRY retrying retry_count"
+    )
+    assert pack_tools_module._clean_passage("  * Café retry-policy: CAFÉ\n1. retry-policy.retry-policy") == (
+        "Café retry-policy: CAFÉ 1. retry-policy.retry-policy"
+    )
+    assert pack_tools_module._clean_passage("a  b\x1c c\n") == "a b c"
+    cases = [
+        (
+            ["retry", "backoff"],
+            "retry backoff",
+            "# Retry Guide",
+            "## Backoff",
+            "https://x.example/retry-backoff",
+            content,
+        ),
+        (["c++", "a.b"], "c++ a.b", "C++ and a.b", "a.b", "u", "c++ c++x a.b a_b"),
+        (["wait"], "wait", "", "", "", ("wait " * 15).strip()),
+    ]
+    scores = [
+        pack_tools_module._search_score(
+            query_terms=terms, phrase=phrase, title=title, heading=heading, url=url, content=text
+        )
+        for terms, phrase, title, heading, url, text in cases
+    ]
+    assert scores == [(48, ["retry", "backoff"]), (24, ["c++", "a.b"]), (30, ["wait"])]