diff --git a/src/docpull/pack_tools.py b/src/docpull/pack_tools.py index 2958e9e..c01e88a 100644 --- a/src/docpull/pack_tools.py +++ b/src/docpull/pack_tools.py @@ -8,7 +8,7 @@ import re from collections import Counter from dataclasses import dataclass, field -from functools import partial +from functools import lru_cache, partial from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -57,6 +57,8 @@ _WORD_RE = re.compile(r"[a-z0-9][a-z0-9-]{2,}", re.IGNORECASE) _MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)]\([^)]+\)") +_LIST_MARKER_RE = re.compile(r"^\s*[-*+]\s+", re.MULTILINE) +_HEADING_MARKER_RE = re.compile(r"^\s{0,3}#{1,6}\s*", re.MULTILINE) _EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") _MONEY_RE = re.compile( r"(? re.Pattern[str]: + return re.compile(rf"\b{re.escape(term)}\b", re.IGNORECASE) + + def _term_count(text: str, term: str) -> int: if not text or not term: return 0 - return len(re.findall(rf"\b{re.escape(term)}\b", text, flags=re.IGNORECASE)) + return len(_term_pattern(term).findall(text)) def _best_search_excerpt(content: str, query_terms: list[str], phrase: str) -> str: @@ -2603,12 +2610,14 @@ def _truncate_text(value: str, max_chars: int) -> str: def _clean_passage(value: str) -> str: - text = _MARKDOWN_LINK_RE.sub(r"\1", value) - text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) - text = re.sub(r"^\s{0,3}#{1,6}\s*", "", text, flags=re.MULTILINE) - text = text.replace("`", "") - text = re.sub(r"\s+", " ", text) - return text.strip() + # Skip each pattern when its required character is absent; most passages have no markdown. + text = _MARKDOWN_LINK_RE.sub(r"\1", value) if "](" in value else value + if "-" in text or "*" in text or "+" in text: + text = _LIST_MARKER_RE.sub("", text) + if "#" in text: + text = _HEADING_MARKER_RE.sub("", text) + # str.split() and regex \s use the same Unicode whitespace set. + return " ".join(text.replace("`", "").split()) def _nearest_sentence(content: str, start: int, end: int) -> str: diff --git a/tests/test_pack_tools.py b/tests/test_pack_tools.py index 9493f59..933bd24 100644 --- a/tests/test_pack_tools.py +++ b/tests/test_pack_tools.py @@ -805,3 +805,37 @@ def test_pack_diff_cli_writes_outputs(tmp_path: Path) -> None: semantic = json.loads((new_pack / "semantic.diff.json").read_text(encoding="utf-8")) assert semantic["summary"]["removed_section"] == 1 assert "Added" in (tmp_path / "diff.md").read_text(encoding="utf-8") + + +def test_search_scoring_matches_pre_fast_path_values() -> None: + # Expected values were recorded from the uncompiled-regex implementation. + content = ( + "- Retry with [backoff](https://b.example) and `retry()`.\n\n" + "### Retry backoff\nRETRY retrying retry_count" + ) + assert pack_tools_module._clean_passage(content) == ( + "Retry with backoff and retry(). Retry backoff RETRY retrying retry_count" + ) + assert pack_tools_module._clean_passage(" * Café retry-policy: CAFÉ\n1. retry-policy.retry-policy") == ( + "Café retry-policy: CAFÉ 1. retry-policy.retry-policy" + ) + assert pack_tools_module._clean_passage("a  b\x1c c\n") == "a b c" + cases = [ + ( + ["retry", "backoff"], + "retry backoff", + "# Retry Guide", + "## Backoff", + "https://x.example/retry-backoff", + content, + ), + (["c++", "a.b"], "c++ a.b", "C++ and a.b", "a.b", "u", "c++ c++x a.b a_b"), + (["wait"], "wait", "", "", "", ("wait " * 15).strip()), + ] + scores = [ + pack_tools_module._search_score( + query_terms=terms, phrase=phrase, title=title, heading=heading, url=url, content=text + ) + for terms, phrase, title, heading, url, text in cases + ] + assert scores == [(48, ["retry", "backoff"]), (24, ["c++", "a.b"]), (30, ["wait"])]