From cc029e57f59ce2178bf24cc696c884211e26766f Mon Sep 17 00:00:00 2001 From: admin-raintree <277948009+admin-raintree@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:37:28 -0700 Subject: [PATCH] Skip redundant regex work when scoring search passages _term_count rebuilt its pattern string on every call, and _clean_passage ran four uncompiled re.sub passes per field even on plain text. Term patterns are now compiled once and cached, cleanup patterns are precompiled and skipped when their required character is absent, and whitespace collapses with str.split, which uses the same Unicode whitespace set as regex \s. Scores and cleaned text are unchanged. Co-Authored-By: Claude Opus 5.5 --- src/docpull/pack_tools.py | 25 +++++++++++++++++-------- tests/test_pack_tools.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/docpull/pack_tools.py b/src/docpull/pack_tools.py index 2958e9e..c01e88a 100644 --- a/src/docpull/pack_tools.py +++ b/src/docpull/pack_tools.py @@ -8,7 +8,7 @@ import re from collections import Counter from dataclasses import dataclass, field -from functools import partial +from functools import lru_cache, partial from pathlib import Path from typing import Any from urllib.parse import urlparse @@ -57,6 +57,8 @@ _WORD_RE = re.compile(r"[a-z0-9][a-z0-9-]{2,}", re.IGNORECASE) _MARKDOWN_LINK_RE = re.compile(r"\[([^\]]+)]\([^)]+\)") +_LIST_MARKER_RE = re.compile(r"^\s*[-*+]\s+", re.MULTILINE) +_HEADING_MARKER_RE = re.compile(r"^\s{0,3}#{1,6}\s*", re.MULTILINE) _EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") _MONEY_RE = re.compile( r"(? re.Pattern[str]: + return re.compile(rf"\b{re.escape(term)}\b", re.IGNORECASE) + + def _term_count(text: str, term: str) -> int: if not text or not term: return 0 - return len(re.findall(rf"\b{re.escape(term)}\b", text, flags=re.IGNORECASE)) + return len(_term_pattern(term).findall(text)) def _best_search_excerpt(content: str, query_terms: list[str], phrase: str) -> str: @@ -2603,12 +2610,14 @@ def _truncate_text(value: str, max_chars: int) -> str: def _clean_passage(value: str) -> str: - text = _MARKDOWN_LINK_RE.sub(r"\1", value) - text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) - text = re.sub(r"^\s{0,3}#{1,6}\s*", "", text, flags=re.MULTILINE) - text = text.replace("`", "") - text = re.sub(r"\s+", " ", text) - return text.strip() + # Skip each pattern when its required character is absent; most passages have no markdown. + text = _MARKDOWN_LINK_RE.sub(r"\1", value) if "](" in value else value + if "-" in text or "*" in text or "+" in text: + text = _LIST_MARKER_RE.sub("", text) + if "#" in text: + text = _HEADING_MARKER_RE.sub("", text) + # str.split() and regex \s use the same Unicode whitespace set. + return " ".join(text.replace("`", "").split()) def _nearest_sentence(content: str, start: int, end: int) -> str: diff --git a/tests/test_pack_tools.py b/tests/test_pack_tools.py index 9493f59..933bd24 100644 --- a/tests/test_pack_tools.py +++ b/tests/test_pack_tools.py @@ -805,3 +805,37 @@ def test_pack_diff_cli_writes_outputs(tmp_path: Path) -> None: semantic = json.loads((new_pack / "semantic.diff.json").read_text(encoding="utf-8")) assert semantic["summary"]["removed_section"] == 1 assert "Added" in (tmp_path / "diff.md").read_text(encoding="utf-8") + + +def test_search_scoring_matches_pre_fast_path_values() -> None: + # Expected values were recorded from the uncompiled-regex implementation. + content = ( + "- Retry with [backoff](https://b.example) and `retry()`.\n\n" + "### Retry backoff\nRETRY retrying retry_count" + ) + assert pack_tools_module._clean_passage(content) == ( + "Retry with backoff and retry(). Retry backoff RETRY retrying retry_count" + ) + assert pack_tools_module._clean_passage(" * Café retry-policy: CAFÉ\n1. retry-policy.retry-policy") == ( + "Café retry-policy: CAFÉ 1. retry-policy.retry-policy" + ) + assert pack_tools_module._clean_passage("a  b\x1c c\n") == "a b c" + cases = [ + ( + ["retry", "backoff"], + "retry backoff", + "# Retry Guide", + "## Backoff", + "https://x.example/retry-backoff", + content, + ), + (["c++", "a.b"], "c++ a.b", "C++ and a.b", "a.b", "u", "c++ c++x a.b a_b"), + (["wait"], "wait", "", "", "", ("wait " * 15).strip()), + ] + scores = [ + pack_tools_module._search_score( + query_terms=terms, phrase=phrase, title=title, heading=heading, url=url, content=text + ) + for terms, phrase, title, heading, url, text in cases + ] + assert scores == [(48, ["retry", "backoff"]), (24, ["c++", "a.b"]), (30, ["wait"])]