From 5b8a12fcc519219d01bbc14f107a8d31ef0b8269 Mon Sep 17 00:00:00 2001 From: admin-raintree <277948009+admin-raintree@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:26:41 -0700 Subject: [PATCH] Add an offline discovery index of downloadable search shards docpull index build turns locked project runs into SQLite FTS shards of page URLs, titles, and headings, with no page bodies, and records each shard's sha256 in manifest.json. docpull index pull downloads shards from an explicit HTTPS manifest URL through the DNS-pinned private-network guard, refuses redirects, enforces a total size limit, and installs only shards that match their sha256. docpull search --backend index and serve --compat --search-backend index search installed shards without sending the query anywhere. Co-Authored-By: Claude Opus 5.5 --- docs/CHANGELOG.md | 5 + docs/alternatives.md | 2 +- docs/cli-recipes.md | 42 +++- docs/surface-contract.md | 3 +- src/docpull/cli.py | 4 + src/docpull/discovery_index.py | 356 +++++++++++++++++++++++++++++++++ src/docpull/server.py | 6 +- src/docpull/surface.py | 3 +- src/docpull/web_search.py | 39 +++- tests/test_discovery_index.py | 158 +++++++++++++++ 10 files changed, 602 insertions(+), 16 deletions(-) create mode 100644 src/docpull/discovery_index.py create mode 100644 tests/test_discovery_index.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 243f027..f4e7698 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--include-content` fetches each result through the local pipeline. `docpull serve --compat --search-backend ...` also serves Tavily and Exa `/search` and Firecrawl `/v1/search` and `/v2/search` shapes. +- Build, download, and search an offline discovery index with + `docpull index build|pull|list` and `docpull search --backend index`. Shards + are SQLite files of page URLs, titles, and headings from synced projects, with + no page bodies. `pull` needs an explicit HTTPS manifest URL. It verifies each + shard's sha256 and enforces a total download size limit. Searches run offline. ### Removed diff --git a/docs/alternatives.md b/docs/alternatives.md index 4d7ffd9..32b5455 100644 --- a/docs/alternatives.md +++ b/docs/alternatives.md @@ -23,7 +23,7 @@ tool. | Self-host a browser-based crawler with LLM-oriented Markdown output | Crawl4AI | Open-source, self-hosted, browser-driven; a good fit when pages need a real browser and you want to run the infrastructure yourself | | Point agent code written for a hosted extract API at a local fetcher | DocPull `serve --compat` | Accepts Tavily extract, Exa contents, and Firecrawl scrape request shapes and answers from the local pipeline; see [Local Extract API](cli-recipes.md#local-extract-api) | | Use a hosted web-to-LLM extraction service | Firecrawl, Jina Reader, hosted extraction APIs | Useful when you want an API service to manage crawling/extraction infrastructure | -| Search the live web before building a pack | DocPull `search` with a SearXNG instance or Marginalia | `docpull search` queries a backend you choose and can fetch each result through the local pipeline; no paid provider is involved. See [Web Search](cli-recipes.md#web-search) | +| Search the live web before building a pack | DocPull `search` with a SearXNG instance, Marginalia, or an offline discovery index | `docpull search` queries a backend you choose and can fetch each result through the local pipeline; no paid provider is involved. See [Web Search](cli-recipes.md#web-search) | ## Where DocPull Is Strongest diff --git a/docs/cli-recipes.md b/docs/cli-recipes.md index fa7c76f..742c822 100644 --- a/docs/cli-recipes.md +++ b/docs/cli-recipes.md @@ -56,8 +56,8 @@ Minimal GitHub Actions step: ## Web Search -`docpull search` queries a search backend you choose explicitly. DocPull has no -index of its own and never calls a paid search provider for this command. +`docpull search` queries a search backend you choose explicitly. It never calls a +paid search provider. ```bash # Self-hosted SearXNG with `json` enabled under search.formats in settings.yml @@ -72,6 +72,7 @@ docpull search "static site generators" --backend marginalia --max-results 5 --j | --- | --- | --- | | `searxng` | Run a SearXNG instance and pass `--searxng-url` or set `DOCPULL_SEARXNG_URL` | Metasearch across the engines your instance enables; the configured URL is trusted, so it may be local | | `marginalia` | None; set `DOCPULL_MARGINALIA_KEY` for your own key | The shared `public` key is often rate limited; results carry Marginalia's license, CC-BY-NC-SA 4.0 for free keys | +| `index` | Download shards with `docpull index pull` | Searches installed discovery shards offline; no query leaves the machine. See [Discovery Index](#discovery-index) | Each query returns at most 20 results. `--include-domain` and `--exclude-domain` filter results by host. One include domain also narrows the upstream query. @@ -79,6 +80,39 @@ filter results by host. One include domain also narrows the upstream query. `docpull sync`, with HTTPS-only fetching, private-network blocking, and robots.txt handling, and attaches the Markdown. +## Discovery Index + +A discovery index is a set of SQLite shards. Each shard lists page URLs, titles, +headings, content hashes, and fetch times from synced DocPull projects. Shards +hold no page bodies. You download shards once, then search them offline with +`--backend index`. Use `--include-content` to fetch hits through the local +pipeline. + +```bash +# Maintainer: build a shard from one or more synced projects +docpull index build --project ./python-docs --name python-docs --out ./published + +# User: download and verify shards, then search offline +docpull index pull https://index.example/v1/manifest.json +docpull search "asyncio timeouts" --backend index +docpull index list +``` + +`build` writes `.sqlite` and adds its sha256, size, and domains to +`manifest.json` in the output directory. Shard URLs in the manifest are relative +to the manifest. + +`pull` requires an HTTPS manifest URL. Pass it as an argument or set +`DOCPULL_INDEX_URL`. DocPull has no default index URL, so nothing downloads +unless you ask. Downloads use the same private-network blocking as page fetches +and refuse redirects. Each shard must match its manifest sha256 before it is +installed. `pull` skips shards that are already current. Before downloading, +`pull` checks the total size against `--max-bytes`, which defaults to 500 MB. + +Shards install to `DOCPULL_INDEX_DIR`, or `$XDG_DATA_HOME/docpull/index` when +that is unset. Pass `--index-dir` to `index`, `search`, or `serve` to use +another directory. + ## Local Extract API `docpull serve --compat` accepts the request shapes of common hosted extract @@ -96,8 +130,8 @@ docpull serve --compat --port 8765 | `POST /contents` | Exa contents: `urls` or `ids`, `text` (bool or `maxCharacters`) | `results[].text`, `statuses[]` | | `POST /v1/scrape`, `POST /v2/scrape` | Firecrawl scrape: `url`, `formats: ["markdown"]` | `success`, `data.markdown`, `data.metadata` | -Start the server with `--search-backend searxng` or `--search-backend marginalia` -to also serve search shapes. Those are Tavily or Exa `POST /search`, chosen by the +Start the server with `--search-backend searxng`, `marginalia`, or `index` to +also serve search shapes. Those are Tavily or Exa `POST /search`, chosen by the request fields, and Firecrawl `POST /v1/search` and `/v2/search`. A client can pin a flavor by setting its base URL to `http://127.0.0.1:8765/tavily`, `/exa`, or `/firecrawl`. Without a search backend, the search routes return diff --git a/docs/surface-contract.md b/docs/surface-contract.md index 1dd1faf..6d0a988 100644 --- a/docs/surface-contract.md +++ b/docs/surface-contract.md @@ -42,7 +42,8 @@ DocPull targets capability alignment, not 1:1 flag parity. MCP should not mirror | List configured sources | Not a primary CLI command | `docpull.mcp.sources` internals, not public SDK | `list_sources` | MCP-focused | | List cached/indexed sources | Not a primary CLI command | Local filesystem/search helpers | `list_indexed` | MCP-focused | | Project lifecycle | `docpull init`, `add`, `install`, `deps`, `sources`, `sync`, `diff`, `status`, `history`, `review`, `release context-pack`, `watch` with explicit crawl bounds | `docpull.project` helpers | Not exposed yet | Core-aligned | -| Search the web | `docpull search QUERY --backend searxng\|marginalia` with domain filters and `--include-content` | `search_web`, `SearchResponse`, `SearchHit` in `docpull.web_search` | Not exposed | CLI-first | +| Search the web | `docpull search QUERY --backend searxng\|marginalia\|index` with domain filters and `--include-content` | `search_web`, `SearchResponse`, `SearchHit` in `docpull.web_search` | Not exposed | CLI-first | +| Offline discovery index | `docpull index build\|pull\|list` | `build_shard`, `pull`, `search_index` in `docpull.discovery_index` | Not exposed | CLI-first | | Search cached sources | SQLite/local search helpers where applicable | `search_sqlite_documents` | `grep_docs` | Adapted | | Query locked context with budgets, version filters, deduplication, and precise citations | `docpull query` | `query_context`, `ContextQueryError` | `context_query` | Core-aligned | | Read cached source by path/range | Filesystem responsibility | Filesystem responsibility | `read_doc` | MCP-specific | diff --git a/src/docpull/cli.py b/src/docpull/cli.py index 856f6e1..e28e710 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -1687,6 +1687,10 @@ def main(argv: list[str] | None = None) -> int: from .graph import run_graph_cli return run_graph_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] == "index": + from .discovery_index import run_index_cli + + return run_index_cli(raw_argv[1:]) if raw_argv and raw_argv[0] == "search": from .web_search import run_search_cli diff --git a/src/docpull/discovery_index.py b/src/docpull/discovery_index.py new file mode 100644 index 0000000..7c04d16 --- /dev/null +++ b/src/docpull/discovery_index.py @@ -0,0 +1,356 @@ +"""Discovery index: downloadable SQLite shards of page URLs, titles, and headings. + +Maintainers build shards from synced DocPull projects. Users download shards once, +verify each against the manifest's sha256, and search them offline, so queries +never leave the machine. Shards hold no page bodies; fetch a hit through the local +pipeline to read it. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import re +import sqlite3 +import sys +import tempfile +from contextlib import closing +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlsplit + +import aiohttp + +from . import __version__ +from .context_query import _STOPWORDS +from .conversion.chunking import _split_on_headings, _strip_frontmatter +from .http.client import _ValidatedResolver +from .project import ( + ProjectError, + _read_context_lock, + _read_jsonl, + _run_dir_for_id, + find_project_root, + project_paths, +) +from .security.url_validator import UrlValidator +from .web_search import SearchHit, WebSearchError, _allowed + +SCHEMA_VERSION = 1 +INDEX_DIR_ENV = "DOCPULL_INDEX_DIR" +INDEX_URL_ENV = "DOCPULL_INDEX_URL" +MANIFEST_NAME = "manifest.json" +MAX_MANIFEST_BYTES = 1_000_000 +DEFAULT_MAX_BYTES = 500_000_000 +_NAME = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") +_TIMEOUT = aiohttp.ClientTimeout(total=600, sock_read=60) + + +class DiscoveryIndexError(ValueError): + """Invalid shard, manifest, or download.""" + + +def default_index_dir() -> Path: + configured = os.environ.get(INDEX_DIR_ENV) + if configured: + return Path(configured).expanduser() + data_home = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share") + return Path(data_home) / "docpull" / "index" + + +def _write_atomic(target: Path, data: bytes) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent, suffix=".tmp", delete=False) as handle: + handle.write(data) + temporary = Path(handle.name) + try: + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _headings(content: str) -> list[str]: + _, body = _strip_frontmatter(content) + return [heading.lstrip("#").strip() for heading, _ in _split_on_headings(body) if heading] + + +def _locked_rows(root: Path) -> list[tuple[str, str, str, str, str, str | None, str | None]]: + paths = project_paths(root) + lock = _read_context_lock(paths.context_lock) + if not lock or not isinstance(lock.get("run_id"), str): + raise DiscoveryIndexError(f"No locked run in {root}; run `docpull sync` first") + _, run_dir = _run_dir_for_id(paths, lock["run_id"]) + rows = [] + for record in _read_jsonl(run_dir / "documents.jsonl"): + url = record.get("url") + content = record.get("content") + if not isinstance(url, str) or urlsplit(url).scheme != "https" or not isinstance(content, str): + continue + metadata = record.get("metadata") + source = metadata.get("docpull_project_source") if isinstance(metadata, dict) else None + rows.append( + ( + url, + str(record.get("title") or ""), + "\n".join(_headings(content)), + urlsplit(url).hostname or "", + str(record.get("content_hash") or ""), + record.get("fetched_at") if isinstance(record.get("fetched_at"), str) else None, + source if isinstance(source, str) else None, + ) + ) + return rows + + +def build_shard(project_roots: list[Path], out_dir: Path, name: str) -> dict[str, Any]: + """Write ``.sqlite`` from locked project runs and record it in ``manifest.json``.""" + if not _NAME.fullmatch(name): + raise DiscoveryIndexError("shard name must be 1-64 lowercase letters, digits, '.', '_' or '-'") + if not project_roots: + raise DiscoveryIndexError("build needs at least one --project") + rows: dict[str, tuple[str, str, str, str, str, str | None, str | None]] = {} + try: + for root in project_roots: + for row in _locked_rows(find_project_root(root)): + rows.setdefault(row[0], row) + except ProjectError as err: + raise DiscoveryIndexError(str(err)) from err + if not rows: + raise DiscoveryIndexError("No HTTPS documents found in the locked runs") + out_dir.mkdir(parents=True, exist_ok=True) + target = out_dir / f"{name}.sqlite" + with tempfile.NamedTemporaryFile(dir=out_dir, suffix=".tmp", delete=False) as handle: + temporary = Path(handle.name) + try: + with closing(sqlite3.connect(temporary)) as conn, conn: + conn.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + conn.execute( + "CREATE TABLE pages (url TEXT PRIMARY KEY, title TEXT, headings TEXT, host TEXT, " + "content_hash TEXT, fetched_at TEXT, source TEXT)" + ) + conn.execute( + "CREATE VIRTUAL TABLE pages_fts USING fts5(title, headings, url, content='pages', " + "content_rowid='rowid', tokenize='unicode61 remove_diacritics 2')" + ) + conn.executemany( + "INSERT INTO meta VALUES (?, ?)", [("name", name), ("docpull_version", __version__)] + ) + conn.executemany("INSERT INTO pages VALUES (?, ?, ?, ?, ?, ?, ?)", sorted(rows.values())) + conn.execute("INSERT INTO pages_fts(pages_fts) VALUES ('rebuild')") + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + with closing(sqlite3.connect(temporary)) as conn: + conn.execute("VACUUM") + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + entry = { + "name": name, + "url": target.name, + "sha256": _sha256_file(target), + "bytes": target.stat().st_size, + "documents": len(rows), + "domains": sorted({row[3] for row in rows.values() if row[3]}), + } + manifest_path = out_dir / MANIFEST_NAME + manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {} + shards = [s for s in manifest.get("shards", []) if isinstance(s, dict) and s.get("name") != name] + manifest = {"schema_version": SCHEMA_VERSION, "shards": sorted([*shards, entry], key=lambda s: s["name"])} + _write_atomic(manifest_path, (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()) + return entry + + +async def _fetch_bytes(url: str, limit: int) -> bytes: + """GET an HTTPS URL through the DNS-pinned private-network guard, refusing redirects.""" + validator = UrlValidator(allowed_schemes={"https"}) + result = validator.validate(url) + if not result.is_valid: + raise DiscoveryIndexError(f"Refusing {url}: {result.rejection_reason}") + headers = {"User-Agent": f"docpull/{__version__} (+https://github.com/raintree-technology/docpull)"} + connector = aiohttp.TCPConnector(resolver=_ValidatedResolver(validator)) + try: + async with ( + aiohttp.ClientSession(connector=connector, timeout=_TIMEOUT, headers=headers) as session, + session.get(url, allow_redirects=False) as response, + ): + if response.status != 200: + raise DiscoveryIndexError(f"{url} returned HTTP {response.status}") + if (response.content_length or 0) > limit: + raise DiscoveryIndexError(f"{url} is larger than the {limit}-byte limit") + chunks: list[bytes] = [] + total = 0 + async for chunk in response.content.iter_chunked(1024 * 1024): + total += len(chunk) + if total > limit: + raise DiscoveryIndexError(f"{url} is larger than the {limit}-byte limit") + chunks.append(chunk) + return b"".join(chunks) + except (aiohttp.ClientError, OSError, TimeoutError) as err: + raise DiscoveryIndexError(f"Download failed for {url}: {type(err).__name__}: {err}") from err + + +def _manifest_shards(manifest: Any, manifest_url: str) -> list[dict[str, Any]]: + if not isinstance(manifest, dict) or manifest.get("schema_version") != SCHEMA_VERSION: + raise DiscoveryIndexError(f"Unsupported discovery manifest; expected schema_version {SCHEMA_VERSION}") + shards = manifest.get("shards") + if not isinstance(shards, list): + raise DiscoveryIndexError("Discovery manifest has no shard list") + checked = [] + for shard in shards: + if ( + not isinstance(shard, dict) + or not isinstance(shard.get("name"), str) + or not _NAME.fullmatch(shard["name"]) + or not isinstance(shard.get("url"), str) + or not re.fullmatch(r"[0-9a-f]{64}", str(shard.get("sha256"))) + or type(shard.get("bytes")) is not int + or shard["bytes"] < 1 + ): + raise DiscoveryIndexError(f"Invalid shard entry in manifest: {shard!r}") + checked.append({**shard, "url": urljoin(manifest_url, shard["url"])}) + return checked + + +async def pull( + manifest_url: str, + index_dir: Path | None = None, + *, + shards: list[str] | None = None, + max_bytes: int = DEFAULT_MAX_BYTES, +) -> dict[str, Any]: + """Download manifest shards that are missing or stale, verifying each sha256.""" + index_dir = index_dir or default_index_dir() + raw = await _fetch_bytes(manifest_url, MAX_MANIFEST_BYTES) + try: + entries = _manifest_shards(json.loads(raw), manifest_url) + except json.JSONDecodeError as err: + raise DiscoveryIndexError("Discovery manifest is not valid JSON") from err + if shards: + unknown = sorted(set(shards) - {entry["name"] for entry in entries}) + if unknown: + raise DiscoveryIndexError(f"Manifest has no shard named: {', '.join(unknown)}") + entries = [entry for entry in entries if entry["name"] in shards] + targets = [(entry, index_dir / f"{entry['name']}.sqlite") for entry in entries] + pending = [(e, t) for e, t in targets if not (t.is_file() and _sha256_file(t) == e["sha256"])] + needed = sum(entry["bytes"] for entry, _ in pending) + if needed > max_bytes: + raise DiscoveryIndexError( + f"Shards need {needed} bytes, over the {max_bytes}-byte limit; raise --max-bytes" + ) + downloaded = [] + for entry, target in pending: + data = await _fetch_bytes(entry["url"], entry["bytes"]) + if hashlib.sha256(data).hexdigest() != entry["sha256"]: + raise DiscoveryIndexError(f"sha256 mismatch for shard {entry['name']}; nothing installed for it") + _write_atomic(target, data) + downloaded.append(entry["name"]) + local = {"schema_version": SCHEMA_VERSION, "source": manifest_url, "shards": [e for e, _ in targets]} + _write_atomic(index_dir / MANIFEST_NAME, (json.dumps(local, indent=2, sort_keys=True) + "\n").encode()) + return { + "index_dir": str(index_dir), + "downloaded": downloaded, + "up_to_date": [e["name"] for e, _ in targets if e["name"] not in downloaded], + "bytes": needed, + } + + +def search_index( + query: str, + *, + index_dir: Path | None = None, + max_results: int = 10, + include_domains: list[str] | None = None, + exclude_domains: list[str] | None = None, +) -> list[SearchHit]: + """Rank installed shards' pages by title, heading, and URL matches. Never touches the network.""" + index_dir = index_dir or default_index_dir() + shard_paths = sorted(index_dir.glob("*.sqlite")) if index_dir.is_dir() else [] + if not shard_paths: + raise WebSearchError(f"No discovery index in {index_dir}; run `docpull index pull `") + terms = sorted(set(re.findall(r"[^\W_]+(?:[_-][^\W_]+)*", query.casefold())) - _STOPWORDS) + if not terms: + raise WebSearchError("query must contain a searchable term") + match = " OR ".join(f'"{term}"' for term in terms) + include, exclude = include_domains or [], exclude_domains or [] + # Overfetch so domain filters still leave max_results hits. + limit = max_results * 5 if include or exclude else max_results + scored: dict[str, tuple[float, SearchHit]] = {} + for path in shard_paths: + try: + with closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro&immutable=1", uri=True)) as conn: + if conn.execute("PRAGMA user_version").fetchone()[0] != SCHEMA_VERSION: + continue + # ponytail: bm25 depends on each shard's own statistics, so cross-shard + # order is approximate; merge into one shard if exact global ranking matters. + rows = conn.execute( + "SELECT p.url, p.title, p.headings, bm25(pages_fts, 5.0, 2.0, 1.0) AS rank " + "FROM pages_fts JOIN pages p ON p.rowid = pages_fts.rowid " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT ?", + (match, limit), + ).fetchall() + except sqlite3.Error as err: + raise WebSearchError(f"Unreadable discovery shard {path.name}: {err}") from err + for url, title, headings, rank in rows: + if not _allowed(url, include, exclude): + continue + hit = SearchHit( + url=url, title=title or "", snippet=" · ".join((headings or "").splitlines()[:8]), score=-rank + ) + if url not in scored or scored[url][0] < -rank: + scored[url] = (-rank, hit) + ranked = sorted(scored.values(), key=lambda item: (-item[0], item[1].url)) + return [hit for _, hit in ranked[:max_results]] + + +def run_index_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="docpull index", description=__doc__.splitlines()[0]) + parser.add_argument( + "--index-dir", type=Path, help=f"Installed shard directory (default: ${INDEX_DIR_ENV})" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + build = subparsers.add_parser("build", help="Build a shard from synced projects") + build.add_argument("--project", action="append", type=Path, required=True, dest="projects") + build.add_argument("--out", type=Path, required=True, help="Directory for the shard and manifest.json") + build.add_argument("--name", required=True, help="Shard name, e.g. python-docs") + pull_parser = subparsers.add_parser("pull", help="Download and verify shards from a manifest URL") + pull_parser.add_argument( + "manifest_url", nargs="?", help=f"HTTPS manifest URL (default: ${INDEX_URL_ENV})" + ) + pull_parser.add_argument("--shard", action="append", dest="shards", help="Only this shard (repeatable)") + pull_parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + subparsers.add_parser("list", help="List installed shards") + args = parser.parse_args(argv) + index_dir = args.index_dir or default_index_dir() + try: + if args.command == "build": + entry = build_shard(args.projects, args.out, args.name) + print(f"Built {entry['name']}: {entry['documents']} pages, {entry['bytes']} bytes -> {args.out}") + elif args.command == "pull": + url = args.manifest_url or os.environ.get(INDEX_URL_ENV) + if not url: + raise DiscoveryIndexError(f"pull needs a manifest URL or {INDEX_URL_ENV}") + result = asyncio.run(pull(url, index_dir, shards=args.shards, max_bytes=args.max_bytes)) + print( + f"Downloaded {len(result['downloaded'])} shard(s), " + f"{len(result['up_to_date'])} already current -> {result['index_dir']}" + ) + else: + for path in sorted(index_dir.glob("*.sqlite")) if index_dir.is_dir() else []: + print(f"{path.stem}\t{path.stat().st_size} bytes") + except DiscoveryIndexError as err: + print(f"docpull index: {err}", file=sys.stderr) + return 1 + return 0 + + +__all__ = ["DiscoveryIndexError", "build_shard", "default_index_dir", "pull", "run_index_cli", "search_index"] diff --git a/src/docpull/server.py b/src/docpull/server.py index c987c6d..035f29d 100644 --- a/src/docpull/server.py +++ b/src/docpull/server.py @@ -127,10 +127,13 @@ def create_serve_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--search-backend", - choices=("searxng", "marginalia"), + choices=("searxng", "marginalia", "index"), help="With --compat, serve /search routes through this backend", ) parser.add_argument("--searxng-url", help="With --search-backend searxng, the SearXNG base URL") + parser.add_argument( + "--index-dir", type=Path, help="With --search-backend index, the installed shard directory" + ) parser.add_argument( "--api-key-env", metavar="NAME", @@ -206,6 +209,7 @@ async def search( exclude_domains=exclude, include_content=content, searxng_url=args.searxng_url, + index_dir=args.index_dir, ) uvicorn.run(CompatASGIApp(search=search, api_key=api_key), host=args.host, port=args.port) diff --git a/src/docpull/surface.py b/src/docpull/surface.py index ebe7651..51db67d 100644 --- a/src/docpull/surface.py +++ b/src/docpull/surface.py @@ -35,7 +35,8 @@ class CliCommand: CliCommand("watch", "Sync, diff, and export one source"), CliCommand("parse", "Parse local PDF/DOCX/text files into a v3 pack"), CliCommand("render", "Render one public URL or check optional browser backends"), - CliCommand("search", "Search the web through self-hosted SearXNG or Marginalia"), + CliCommand("search", "Search the web through self-hosted SearXNG, Marginalia, or a discovery index"), + CliCommand("index", "Build, download, and list offline discovery index shards"), CliCommand("openapi-pack", "Build a local v3 pack from an OpenAPI spec"), CliCommand("feed-pack", "Build a local v3 pack from an RSS, Atom, JSON Feed, or feed page"), CliCommand("paper-pack", "Build a local v3 pack from papers and scholarly metadata"), diff --git a/src/docpull/web_search.py b/src/docpull/web_search.py index bc80240..5640919 100644 --- a/src/docpull/web_search.py +++ b/src/docpull/web_search.py @@ -1,9 +1,11 @@ -"""Web search through backends the user chooses: self-hosted SearXNG or Marginalia. - -DocPull has no search index of its own. ``search_web`` queries a backend the -caller names explicitly, never a paid provider, then optionally fetches the top -results through the local fetch pipeline so content keeps DocPull's SSRF and -robots.txt protections. +"""Web search through backends the user chooses: self-hosted SearXNG, Marginalia, or +an offline discovery index. + +``search_web`` queries a backend the caller names explicitly, never a paid +provider, then optionally fetches the top results through the local fetch +pipeline so content keeps DocPull's SSRF and robots.txt protections. The +``index`` backend searches downloaded discovery shards and sends no query +over the network. """ from __future__ import annotations @@ -14,6 +16,7 @@ import os import sys from dataclasses import asdict, dataclass, field, replace +from pathlib import Path from typing import Any from urllib.parse import urlsplit @@ -21,7 +24,7 @@ from . import __version__ -BACKENDS = ("searxng", "marginalia") +BACKENDS = ("searxng", "marginalia", "index") MAX_RESULTS = 20 SEARXNG_URL_ENV = "DOCPULL_SEARXNG_URL" MARGINALIA_KEY_ENV = "DOCPULL_MARGINALIA_KEY" @@ -67,6 +70,7 @@ async def search_web( include_content: bool = False, searxng_url: str | None = None, marginalia_key: str | None = None, + index_dir: Path | None = None, ) -> SearchResponse: """Search the web with an explicitly chosen backend.""" query = query.strip() @@ -79,6 +83,20 @@ async def search_web( include = [_domain(item) for item in include_domains or []] exclude = [_domain(item) for item in exclude_domains or []] + if backend == "index": + from .discovery_index import search_index + + hits = search_index( + query, + index_dir=index_dir, + max_results=max_results, + include_domains=include, + exclude_domains=exclude, + ) + if include_content: + hits = await _attach_content(hits) + return SearchResponse(query=query, backend=backend, hits=hits) + key = marginalia_key or os.environ.get(MARGINALIA_KEY_ENV) or "public" headers = {"User-Agent": f"docpull/{__version__} (+https://github.com/raintree-technology/docpull)"} async with aiohttp.ClientSession(timeout=_TIMEOUT, headers=headers) as session: @@ -237,7 +255,10 @@ async def _attach_content(hits: list[SearchHit]) -> list[SearchHit]: def create_search_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="docpull search", - description="Search the web through self-hosted SearXNG or Marginalia; no paid provider is used", + description=( + "Search the web through self-hosted SearXNG, Marginalia, or an offline discovery index; " + "no paid provider is used" + ), ) parser.add_argument("query") parser.add_argument("--backend", choices=BACKENDS, required=True) @@ -250,6 +271,7 @@ def create_search_parser() -> argparse.ArgumentParser: help="Fetch each result through the local pipeline and include its Markdown", ) parser.add_argument("--searxng-url", help=f"SearXNG base URL (default: ${SEARXNG_URL_ENV})") + parser.add_argument("--index-dir", type=Path, help="With --backend index, the installed shard directory") parser.add_argument("--json", action="store_true", dest="json_output") return parser @@ -266,6 +288,7 @@ def run_search_cli(argv: list[str] | None = None) -> int: exclude_domains=args.exclude_domains, include_content=args.include_content, searxng_url=args.searxng_url, + index_dir=args.index_dir, ) ) except WebSearchError as err: diff --git a/tests/test_discovery_index.py b/tests/test_discovery_index.py new file mode 100644 index 0000000..8c037ce --- /dev/null +++ b/tests/test_discovery_index.py @@ -0,0 +1,158 @@ +"""Discovery shards: build from locked runs, verified download, and offline search.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest + +from docpull import discovery_index +from docpull import project as project_module +from docpull.cli import main +from docpull.discovery_index import DiscoveryIndexError, build_shard, pull, search_index +from docpull.project import init_project, sync_project +from docpull.web_search import WebSearchError, search_web + +PAGES = { + "https://docs.alpha.example/retry": ( + "Retry policies", + "# Retry policies\n\n## Backoff\n\nSECRET BODY TEXT", + ), + "https://docs.alpha.example/auth": ("Authentication", "# Authentication\n\n## Retry on 401\n\nbody"), + "https://blog.beta.example/post": ("Release notes", "# Release notes\n\n```\n# not a heading\n```\n"), +} + + +def _project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "proj" + init_project(name="proj", source="https://docs.alpha.example/", root=root) + + async def fetch(**kwargs: Any) -> dict[str, Any]: + records = [ + { + "document_id": f"doc_{i}", + "url": url, + "title": title, + "content": content, + "content_hash": hashlib.sha256(content.encode()).hexdigest(), + "fetched_at": "2026-10-12T00:00:00Z", + "metadata": {"docpull_project_source": kwargs["source"].name}, + } + for i, (url, (title, content)) in enumerate(PAGES.items()) + ] + return {"records": records, "errors": [], "skips": [], "stats": {}} + + monkeypatch.setattr(project_module, "_sync_source", fetch) + sync_project(root=root, run_id="proj_run") + return root + + +@pytest.fixture +def shards(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + out = tmp_path / "published" + build_shard([_project(tmp_path, monkeypatch)], out, "alpha") + return out + + +def test_build_writes_titles_and_headings_but_no_bodies(shards: Path) -> None: + manifest = json.loads((shards / "manifest.json").read_text()) + [entry] = manifest["shards"] + shard = shards / "alpha.sqlite" + assert entry["sha256"] == hashlib.sha256(shard.read_bytes()).hexdigest() + assert entry["documents"] == 3 and entry["domains"] == ["blog.beta.example", "docs.alpha.example"] + with sqlite3.connect(shard) as conn: + rows = dict(conn.execute("SELECT url, headings FROM pages").fetchall()) + assert rows["https://docs.alpha.example/retry"] == "Retry policies\nBackoff" + assert rows["https://blog.beta.example/post"] == "Release notes" + assert b"SECRET BODY TEXT" not in shard.read_bytes() + + +def _serve(monkeypatch: pytest.MonkeyPatch, files: dict[str, bytes]) -> list[str]: + fetched: list[str] = [] + + async def fake_fetch(url: str, limit: int) -> bytes: + fetched.append(url) + data = files[url] + if len(data) > limit: + raise DiscoveryIndexError("too large") + return data + + monkeypatch.setattr(discovery_index, "_fetch_bytes", fake_fetch) + return fetched + + +def test_pull_verifies_and_skips_current_shards( + shards: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + base = "https://index.example/v1/" + fetched = _serve( + monkeypatch, + { + base + "manifest.json": (shards / "manifest.json").read_bytes(), + base + "alpha.sqlite": (shards / "alpha.sqlite").read_bytes(), + }, + ) + installed = tmp_path / "installed" + first = asyncio.run(pull(base + "manifest.json", installed)) + assert first["downloaded"] == ["alpha"] and (installed / "alpha.sqlite").is_file() + second = asyncio.run(pull(base + "manifest.json", installed)) + assert second["downloaded"] == [] and second["up_to_date"] == ["alpha"] + assert fetched.count(base + "alpha.sqlite") == 1 + + with pytest.raises(DiscoveryIndexError, match="limit"): + asyncio.run(pull(base + "manifest.json", tmp_path / "small", max_bytes=10)) + with pytest.raises(DiscoveryIndexError, match="no shard named"): + asyncio.run(pull(base + "manifest.json", installed, shards=["missing"])) + + +def test_pull_rejects_tampered_shard(shards: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + base = "https://index.example/" + data = bytearray((shards / "alpha.sqlite").read_bytes()) + data[-1] ^= 1 + _serve( + monkeypatch, + {base + "manifest.json": (shards / "manifest.json").read_bytes(), base + "alpha.sqlite": bytes(data)}, + ) + with pytest.raises(DiscoveryIndexError, match="sha256 mismatch"): + asyncio.run(pull(base + "manifest.json", tmp_path / "installed")) + assert not (tmp_path / "installed" / "alpha.sqlite").exists() + + +@pytest.mark.parametrize("url", ["http://index.example/manifest.json", "https://127.0.0.1/manifest.json"]) +def test_fetch_refuses_plain_http_and_private_hosts(url: str) -> None: + with pytest.raises(DiscoveryIndexError, match="Refusing"): + asyncio.run(discovery_index._fetch_bytes(url, 100)) + + +def test_index_backend_searches_offline(shards: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def no_network(*args: Any, **kwargs: Any) -> None: + pytest.fail("index search attempted network access") + + monkeypatch.setattr(socket, "create_connection", no_network) + response = asyncio.run(search_web("retry", backend="index", index_dir=shards)) + assert [hit.url for hit in response.hits] == [ + "https://docs.alpha.example/retry", + "https://docs.alpha.example/auth", + ] + assert response.hits[0].snippet == "Retry policies · Backoff" + filtered = search_index("retry release", index_dir=shards, exclude_domains=["docs.alpha.example"]) + assert [hit.url for hit in filtered] == ["https://blog.beta.example/post"] + with pytest.raises(WebSearchError, match="index pull"): + search_index("retry", index_dir=shards / "missing") + + +def test_index_cli_build_and_list( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + root = _project(tmp_path, monkeypatch) + out = tmp_path / "cli" + assert main(["index", "build", "--project", str(root), "--out", str(out), "--name", "alpha"]) == 0 + assert main(["index", "--index-dir", str(out), "list"]) == 0 + assert "alpha\t" in capsys.readouterr().out + assert main(["index", "pull"]) == 1