From 5b8a12fcc519219d01bbc14f107a8d31ef0b8269 Mon Sep 17 00:00:00 2001 From: admin-raintree <277948009+admin-raintree@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:26:41 -0700 Subject: [PATCH 1/2] Add an offline discovery index of downloadable search shards docpull index build turns locked project runs into SQLite FTS shards of page URLs, titles, and headings, with no page bodies, and records each shard's sha256 in manifest.json. docpull index pull downloads shards from an explicit HTTPS manifest URL through the DNS-pinned private-network guard, refuses redirects, enforces a total size limit, and installs only shards that match their sha256. docpull search --backend index and serve --compat --search-backend index search installed shards without sending the query anywhere. Co-Authored-By: Claude Opus 5.5 --- docs/CHANGELOG.md | 5 + docs/alternatives.md | 2 +- docs/cli-recipes.md | 42 +++- docs/surface-contract.md | 3 +- src/docpull/cli.py | 4 + src/docpull/discovery_index.py | 356 +++++++++++++++++++++++++++++++++ src/docpull/server.py | 6 +- src/docpull/surface.py | 3 +- src/docpull/web_search.py | 39 +++- tests/test_discovery_index.py | 158 +++++++++++++++ 10 files changed, 602 insertions(+), 16 deletions(-) create mode 100644 src/docpull/discovery_index.py create mode 100644 tests/test_discovery_index.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 243f027..f4e7698 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -19,6 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--include-content` fetches each result through the local pipeline. `docpull serve --compat --search-backend ...` also serves Tavily and Exa `/search` and Firecrawl `/v1/search` and `/v2/search` shapes. +- Build, download, and search an offline discovery index with + `docpull index build|pull|list` and `docpull search --backend index`. Shards + are SQLite files of page URLs, titles, and headings from synced projects, with + no page bodies. `pull` needs an explicit HTTPS manifest URL. It verifies each + shard's sha256 and enforces a total download size limit. Searches run offline. ### Removed diff --git a/docs/alternatives.md b/docs/alternatives.md index 4d7ffd9..32b5455 100644 --- a/docs/alternatives.md +++ b/docs/alternatives.md @@ -23,7 +23,7 @@ tool. | Self-host a browser-based crawler with LLM-oriented Markdown output | Crawl4AI | Open-source, self-hosted, browser-driven; a good fit when pages need a real browser and you want to run the infrastructure yourself | | Point agent code written for a hosted extract API at a local fetcher | DocPull `serve --compat` | Accepts Tavily extract, Exa contents, and Firecrawl scrape request shapes and answers from the local pipeline; see [Local Extract API](cli-recipes.md#local-extract-api) | | Use a hosted web-to-LLM extraction service | Firecrawl, Jina Reader, hosted extraction APIs | Useful when you want an API service to manage crawling/extraction infrastructure | -| Search the live web before building a pack | DocPull `search` with a SearXNG instance or Marginalia | `docpull search` queries a backend you choose and can fetch each result through the local pipeline; no paid provider is involved. See [Web Search](cli-recipes.md#web-search) | +| Search the live web before building a pack | DocPull `search` with a SearXNG instance, Marginalia, or an offline discovery index | `docpull search` queries a backend you choose and can fetch each result through the local pipeline; no paid provider is involved. See [Web Search](cli-recipes.md#web-search) | ## Where DocPull Is Strongest diff --git a/docs/cli-recipes.md b/docs/cli-recipes.md index fa7c76f..742c822 100644 --- a/docs/cli-recipes.md +++ b/docs/cli-recipes.md @@ -56,8 +56,8 @@ Minimal GitHub Actions step: ## Web Search -`docpull search` queries a search backend you choose explicitly. DocPull has no -index of its own and never calls a paid search provider for this command. +`docpull search` queries a search backend you choose explicitly. It never calls a +paid search provider. ```bash # Self-hosted SearXNG with `json` enabled under search.formats in settings.yml @@ -72,6 +72,7 @@ docpull search "static site generators" --backend marginalia --max-results 5 --j | --- | --- | --- | | `searxng` | Run a SearXNG instance and pass `--searxng-url` or set `DOCPULL_SEARXNG_URL` | Metasearch across the engines your instance enables; the configured URL is trusted, so it may be local | | `marginalia` | None; set `DOCPULL_MARGINALIA_KEY` for your own key | The shared `public` key is often rate limited; results carry Marginalia's license, CC-BY-NC-SA 4.0 for free keys | +| `index` | Download shards with `docpull index pull` | Searches installed discovery shards offline; no query leaves the machine. See [Discovery Index](#discovery-index) | Each query returns at most 20 results. `--include-domain` and `--exclude-domain` filter results by host. One include domain also narrows the upstream query. @@ -79,6 +80,39 @@ filter results by host. One include domain also narrows the upstream query. `docpull sync`, with HTTPS-only fetching, private-network blocking, and robots.txt handling, and attaches the Markdown. +## Discovery Index + +A discovery index is a set of SQLite shards. Each shard lists page URLs, titles, +headings, content hashes, and fetch times from synced DocPull projects. Shards +hold no page bodies. You download shards once, then search them offline with +`--backend index`. Use `--include-content` to fetch hits through the local +pipeline. + +```bash +# Maintainer: build a shard from one or more synced projects +docpull index build --project ./python-docs --name python-docs --out ./published + +# User: download and verify shards, then search offline +docpull index pull https://index.example/v1/manifest.json +docpull search "asyncio timeouts" --backend index +docpull index list +``` + +`build` writes `.sqlite` and adds its sha256, size, and domains to +`manifest.json` in the output directory. Shard URLs in the manifest are relative +to the manifest. + +`pull` requires an HTTPS manifest URL. Pass it as an argument or set +`DOCPULL_INDEX_URL`. DocPull has no default index URL, so nothing downloads +unless you ask. Downloads use the same private-network blocking as page fetches +and refuse redirects. Each shard must match its manifest sha256 before it is +installed. `pull` skips shards that are already current. Before downloading, +`pull` checks the total size against `--max-bytes`, which defaults to 500 MB. + +Shards install to `DOCPULL_INDEX_DIR`, or `$XDG_DATA_HOME/docpull/index` when +that is unset. Pass `--index-dir` to `index`, `search`, or `serve` to use +another directory. + ## Local Extract API `docpull serve --compat` accepts the request shapes of common hosted extract @@ -96,8 +130,8 @@ docpull serve --compat --port 8765 | `POST /contents` | Exa contents: `urls` or `ids`, `text` (bool or `maxCharacters`) | `results[].text`, `statuses[]` | | `POST /v1/scrape`, `POST /v2/scrape` | Firecrawl scrape: `url`, `formats: ["markdown"]` | `success`, `data.markdown`, `data.metadata` | -Start the server with `--search-backend searxng` or `--search-backend marginalia` -to also serve search shapes. Those are Tavily or Exa `POST /search`, chosen by the +Start the server with `--search-backend searxng`, `marginalia`, or `index` to +also serve search shapes. Those are Tavily or Exa `POST /search`, chosen by the request fields, and Firecrawl `POST /v1/search` and `/v2/search`. A client can pin a flavor by setting its base URL to `http://127.0.0.1:8765/tavily`, `/exa`, or `/firecrawl`. Without a search backend, the search routes return diff --git a/docs/surface-contract.md b/docs/surface-contract.md index 1dd1faf..6d0a988 100644 --- a/docs/surface-contract.md +++ b/docs/surface-contract.md @@ -42,7 +42,8 @@ DocPull targets capability alignment, not 1:1 flag parity. MCP should not mirror | List configured sources | Not a primary CLI command | `docpull.mcp.sources` internals, not public SDK | `list_sources` | MCP-focused | | List cached/indexed sources | Not a primary CLI command | Local filesystem/search helpers | `list_indexed` | MCP-focused | | Project lifecycle | `docpull init`, `add`, `install`, `deps`, `sources`, `sync`, `diff`, `status`, `history`, `review`, `release context-pack`, `watch` with explicit crawl bounds | `docpull.project` helpers | Not exposed yet | Core-aligned | -| Search the web | `docpull search QUERY --backend searxng\|marginalia` with domain filters and `--include-content` | `search_web`, `SearchResponse`, `SearchHit` in `docpull.web_search` | Not exposed | CLI-first | +| Search the web | `docpull search QUERY --backend searxng\|marginalia\|index` with domain filters and `--include-content` | `search_web`, `SearchResponse`, `SearchHit` in `docpull.web_search` | Not exposed | CLI-first | +| Offline discovery index | `docpull index build\|pull\|list` | `build_shard`, `pull`, `search_index` in `docpull.discovery_index` | Not exposed | CLI-first | | Search cached sources | SQLite/local search helpers where applicable | `search_sqlite_documents` | `grep_docs` | Adapted | | Query locked context with budgets, version filters, deduplication, and precise citations | `docpull query` | `query_context`, `ContextQueryError` | `context_query` | Core-aligned | | Read cached source by path/range | Filesystem responsibility | Filesystem responsibility | `read_doc` | MCP-specific | diff --git a/src/docpull/cli.py b/src/docpull/cli.py index 856f6e1..e28e710 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -1687,6 +1687,10 @@ def main(argv: list[str] | None = None) -> int: from .graph import run_graph_cli return run_graph_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] == "index": + from .discovery_index import run_index_cli + + return run_index_cli(raw_argv[1:]) if raw_argv and raw_argv[0] == "search": from .web_search import run_search_cli diff --git a/src/docpull/discovery_index.py b/src/docpull/discovery_index.py new file mode 100644 index 0000000..7c04d16 --- /dev/null +++ b/src/docpull/discovery_index.py @@ -0,0 +1,356 @@ +"""Discovery index: downloadable SQLite shards of page URLs, titles, and headings. + +Maintainers build shards from synced DocPull projects. Users download shards once, +verify each against the manifest's sha256, and search them offline, so queries +never leave the machine. Shards hold no page bodies; fetch a hit through the local +pipeline to read it. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import re +import sqlite3 +import sys +import tempfile +from contextlib import closing +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlsplit + +import aiohttp + +from . import __version__ +from .context_query import _STOPWORDS +from .conversion.chunking import _split_on_headings, _strip_frontmatter +from .http.client import _ValidatedResolver +from .project import ( + ProjectError, + _read_context_lock, + _read_jsonl, + _run_dir_for_id, + find_project_root, + project_paths, +) +from .security.url_validator import UrlValidator +from .web_search import SearchHit, WebSearchError, _allowed + +SCHEMA_VERSION = 1 +INDEX_DIR_ENV = "DOCPULL_INDEX_DIR" +INDEX_URL_ENV = "DOCPULL_INDEX_URL" +MANIFEST_NAME = "manifest.json" +MAX_MANIFEST_BYTES = 1_000_000 +DEFAULT_MAX_BYTES = 500_000_000 +_NAME = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") +_TIMEOUT = aiohttp.ClientTimeout(total=600, sock_read=60) + + +class DiscoveryIndexError(ValueError): + """Invalid shard, manifest, or download.""" + + +def default_index_dir() -> Path: + configured = os.environ.get(INDEX_DIR_ENV) + if configured: + return Path(configured).expanduser() + data_home = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share") + return Path(data_home) / "docpull" / "index" + + +def _write_atomic(target: Path, data: bytes) -> None: + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent, suffix=".tmp", delete=False) as handle: + handle.write(data) + temporary = Path(handle.name) + try: + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _headings(content: str) -> list[str]: + _, body = _strip_frontmatter(content) + return [heading.lstrip("#").strip() for heading, _ in _split_on_headings(body) if heading] + + +def _locked_rows(root: Path) -> list[tuple[str, str, str, str, str, str | None, str | None]]: + paths = project_paths(root) + lock = _read_context_lock(paths.context_lock) + if not lock or not isinstance(lock.get("run_id"), str): + raise DiscoveryIndexError(f"No locked run in {root}; run `docpull sync` first") + _, run_dir = _run_dir_for_id(paths, lock["run_id"]) + rows = [] + for record in _read_jsonl(run_dir / "documents.jsonl"): + url = record.get("url") + content = record.get("content") + if not isinstance(url, str) or urlsplit(url).scheme != "https" or not isinstance(content, str): + continue + metadata = record.get("metadata") + source = metadata.get("docpull_project_source") if isinstance(metadata, dict) else None + rows.append( + ( + url, + str(record.get("title") or ""), + "\n".join(_headings(content)), + urlsplit(url).hostname or "", + str(record.get("content_hash") or ""), + record.get("fetched_at") if isinstance(record.get("fetched_at"), str) else None, + source if isinstance(source, str) else None, + ) + ) + return rows + + +def build_shard(project_roots: list[Path], out_dir: Path, name: str) -> dict[str, Any]: + """Write ``.sqlite`` from locked project runs and record it in ``manifest.json``.""" + if not _NAME.fullmatch(name): + raise DiscoveryIndexError("shard name must be 1-64 lowercase letters, digits, '.', '_' or '-'") + if not project_roots: + raise DiscoveryIndexError("build needs at least one --project") + rows: dict[str, tuple[str, str, str, str, str, str | None, str | None]] = {} + try: + for root in project_roots: + for row in _locked_rows(find_project_root(root)): + rows.setdefault(row[0], row) + except ProjectError as err: + raise DiscoveryIndexError(str(err)) from err + if not rows: + raise DiscoveryIndexError("No HTTPS documents found in the locked runs") + out_dir.mkdir(parents=True, exist_ok=True) + target = out_dir / f"{name}.sqlite" + with tempfile.NamedTemporaryFile(dir=out_dir, suffix=".tmp", delete=False) as handle: + temporary = Path(handle.name) + try: + with closing(sqlite3.connect(temporary)) as conn, conn: + conn.execute("CREATE TABLE meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + conn.execute( + "CREATE TABLE pages (url TEXT PRIMARY KEY, title TEXT, headings TEXT, host TEXT, " + "content_hash TEXT, fetched_at TEXT, source TEXT)" + ) + conn.execute( + "CREATE VIRTUAL TABLE pages_fts USING fts5(title, headings, url, content='pages', " + "content_rowid='rowid', tokenize='unicode61 remove_diacritics 2')" + ) + conn.executemany( + "INSERT INTO meta VALUES (?, ?)", [("name", name), ("docpull_version", __version__)] + ) + conn.executemany("INSERT INTO pages VALUES (?, ?, ?, ?, ?, ?, ?)", sorted(rows.values())) + conn.execute("INSERT INTO pages_fts(pages_fts) VALUES ('rebuild')") + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + with closing(sqlite3.connect(temporary)) as conn: + conn.execute("VACUUM") + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) + entry = { + "name": name, + "url": target.name, + "sha256": _sha256_file(target), + "bytes": target.stat().st_size, + "documents": len(rows), + "domains": sorted({row[3] for row in rows.values() if row[3]}), + } + manifest_path = out_dir / MANIFEST_NAME + manifest = json.loads(manifest_path.read_text()) if manifest_path.is_file() else {} + shards = [s for s in manifest.get("shards", []) if isinstance(s, dict) and s.get("name") != name] + manifest = {"schema_version": SCHEMA_VERSION, "shards": sorted([*shards, entry], key=lambda s: s["name"])} + _write_atomic(manifest_path, (json.dumps(manifest, indent=2, sort_keys=True) + "\n").encode()) + return entry + + +async def _fetch_bytes(url: str, limit: int) -> bytes: + """GET an HTTPS URL through the DNS-pinned private-network guard, refusing redirects.""" + validator = UrlValidator(allowed_schemes={"https"}) + result = validator.validate(url) + if not result.is_valid: + raise DiscoveryIndexError(f"Refusing {url}: {result.rejection_reason}") + headers = {"User-Agent": f"docpull/{__version__} (+https://github.com/raintree-technology/docpull)"} + connector = aiohttp.TCPConnector(resolver=_ValidatedResolver(validator)) + try: + async with ( + aiohttp.ClientSession(connector=connector, timeout=_TIMEOUT, headers=headers) as session, + session.get(url, allow_redirects=False) as response, + ): + if response.status != 200: + raise DiscoveryIndexError(f"{url} returned HTTP {response.status}") + if (response.content_length or 0) > limit: + raise DiscoveryIndexError(f"{url} is larger than the {limit}-byte limit") + chunks: list[bytes] = [] + total = 0 + async for chunk in response.content.iter_chunked(1024 * 1024): + total += len(chunk) + if total > limit: + raise DiscoveryIndexError(f"{url} is larger than the {limit}-byte limit") + chunks.append(chunk) + return b"".join(chunks) + except (aiohttp.ClientError, OSError, TimeoutError) as err: + raise DiscoveryIndexError(f"Download failed for {url}: {type(err).__name__}: {err}") from err + + +def _manifest_shards(manifest: Any, manifest_url: str) -> list[dict[str, Any]]: + if not isinstance(manifest, dict) or manifest.get("schema_version") != SCHEMA_VERSION: + raise DiscoveryIndexError(f"Unsupported discovery manifest; expected schema_version {SCHEMA_VERSION}") + shards = manifest.get("shards") + if not isinstance(shards, list): + raise DiscoveryIndexError("Discovery manifest has no shard list") + checked = [] + for shard in shards: + if ( + not isinstance(shard, dict) + or not isinstance(shard.get("name"), str) + or not _NAME.fullmatch(shard["name"]) + or not isinstance(shard.get("url"), str) + or not re.fullmatch(r"[0-9a-f]{64}", str(shard.get("sha256"))) + or type(shard.get("bytes")) is not int + or shard["bytes"] < 1 + ): + raise DiscoveryIndexError(f"Invalid shard entry in manifest: {shard!r}") + checked.append({**shard, "url": urljoin(manifest_url, shard["url"])}) + return checked + + +async def pull( + manifest_url: str, + index_dir: Path | None = None, + *, + shards: list[str] | None = None, + max_bytes: int = DEFAULT_MAX_BYTES, +) -> dict[str, Any]: + """Download manifest shards that are missing or stale, verifying each sha256.""" + index_dir = index_dir or default_index_dir() + raw = await _fetch_bytes(manifest_url, MAX_MANIFEST_BYTES) + try: + entries = _manifest_shards(json.loads(raw), manifest_url) + except json.JSONDecodeError as err: + raise DiscoveryIndexError("Discovery manifest is not valid JSON") from err + if shards: + unknown = sorted(set(shards) - {entry["name"] for entry in entries}) + if unknown: + raise DiscoveryIndexError(f"Manifest has no shard named: {', '.join(unknown)}") + entries = [entry for entry in entries if entry["name"] in shards] + targets = [(entry, index_dir / f"{entry['name']}.sqlite") for entry in entries] + pending = [(e, t) for e, t in targets if not (t.is_file() and _sha256_file(t) == e["sha256"])] + needed = sum(entry["bytes"] for entry, _ in pending) + if needed > max_bytes: + raise DiscoveryIndexError( + f"Shards need {needed} bytes, over the {max_bytes}-byte limit; raise --max-bytes" + ) + downloaded = [] + for entry, target in pending: + data = await _fetch_bytes(entry["url"], entry["bytes"]) + if hashlib.sha256(data).hexdigest() != entry["sha256"]: + raise DiscoveryIndexError(f"sha256 mismatch for shard {entry['name']}; nothing installed for it") + _write_atomic(target, data) + downloaded.append(entry["name"]) + local = {"schema_version": SCHEMA_VERSION, "source": manifest_url, "shards": [e for e, _ in targets]} + _write_atomic(index_dir / MANIFEST_NAME, (json.dumps(local, indent=2, sort_keys=True) + "\n").encode()) + return { + "index_dir": str(index_dir), + "downloaded": downloaded, + "up_to_date": [e["name"] for e, _ in targets if e["name"] not in downloaded], + "bytes": needed, + } + + +def search_index( + query: str, + *, + index_dir: Path | None = None, + max_results: int = 10, + include_domains: list[str] | None = None, + exclude_domains: list[str] | None = None, +) -> list[SearchHit]: + """Rank installed shards' pages by title, heading, and URL matches. Never touches the network.""" + index_dir = index_dir or default_index_dir() + shard_paths = sorted(index_dir.glob("*.sqlite")) if index_dir.is_dir() else [] + if not shard_paths: + raise WebSearchError(f"No discovery index in {index_dir}; run `docpull index pull `") + terms = sorted(set(re.findall(r"[^\W_]+(?:[_-][^\W_]+)*", query.casefold())) - _STOPWORDS) + if not terms: + raise WebSearchError("query must contain a searchable term") + match = " OR ".join(f'"{term}"' for term in terms) + include, exclude = include_domains or [], exclude_domains or [] + # Overfetch so domain filters still leave max_results hits. + limit = max_results * 5 if include or exclude else max_results + scored: dict[str, tuple[float, SearchHit]] = {} + for path in shard_paths: + try: + with closing(sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro&immutable=1", uri=True)) as conn: + if conn.execute("PRAGMA user_version").fetchone()[0] != SCHEMA_VERSION: + continue + # ponytail: bm25 depends on each shard's own statistics, so cross-shard + # order is approximate; merge into one shard if exact global ranking matters. + rows = conn.execute( + "SELECT p.url, p.title, p.headings, bm25(pages_fts, 5.0, 2.0, 1.0) AS rank " + "FROM pages_fts JOIN pages p ON p.rowid = pages_fts.rowid " + "WHERE pages_fts MATCH ? ORDER BY rank LIMIT ?", + (match, limit), + ).fetchall() + except sqlite3.Error as err: + raise WebSearchError(f"Unreadable discovery shard {path.name}: {err}") from err + for url, title, headings, rank in rows: + if not _allowed(url, include, exclude): + continue + hit = SearchHit( + url=url, title=title or "", snippet=" · ".join((headings or "").splitlines()[:8]), score=-rank + ) + if url not in scored or scored[url][0] < -rank: + scored[url] = (-rank, hit) + ranked = sorted(scored.values(), key=lambda item: (-item[0], item[1].url)) + return [hit for _, hit in ranked[:max_results]] + + +def run_index_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="docpull index", description=__doc__.splitlines()[0]) + parser.add_argument( + "--index-dir", type=Path, help=f"Installed shard directory (default: ${INDEX_DIR_ENV})" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + build = subparsers.add_parser("build", help="Build a shard from synced projects") + build.add_argument("--project", action="append", type=Path, required=True, dest="projects") + build.add_argument("--out", type=Path, required=True, help="Directory for the shard and manifest.json") + build.add_argument("--name", required=True, help="Shard name, e.g. python-docs") + pull_parser = subparsers.add_parser("pull", help="Download and verify shards from a manifest URL") + pull_parser.add_argument( + "manifest_url", nargs="?", help=f"HTTPS manifest URL (default: ${INDEX_URL_ENV})" + ) + pull_parser.add_argument("--shard", action="append", dest="shards", help="Only this shard (repeatable)") + pull_parser.add_argument("--max-bytes", type=int, default=DEFAULT_MAX_BYTES) + subparsers.add_parser("list", help="List installed shards") + args = parser.parse_args(argv) + index_dir = args.index_dir or default_index_dir() + try: + if args.command == "build": + entry = build_shard(args.projects, args.out, args.name) + print(f"Built {entry['name']}: {entry['documents']} pages, {entry['bytes']} bytes -> {args.out}") + elif args.command == "pull": + url = args.manifest_url or os.environ.get(INDEX_URL_ENV) + if not url: + raise DiscoveryIndexError(f"pull needs a manifest URL or {INDEX_URL_ENV}") + result = asyncio.run(pull(url, index_dir, shards=args.shards, max_bytes=args.max_bytes)) + print( + f"Downloaded {len(result['downloaded'])} shard(s), " + f"{len(result['up_to_date'])} already current -> {result['index_dir']}" + ) + else: + for path in sorted(index_dir.glob("*.sqlite")) if index_dir.is_dir() else []: + print(f"{path.stem}\t{path.stat().st_size} bytes") + except DiscoveryIndexError as err: + print(f"docpull index: {err}", file=sys.stderr) + return 1 + return 0 + + +__all__ = ["DiscoveryIndexError", "build_shard", "default_index_dir", "pull", "run_index_cli", "search_index"] diff --git a/src/docpull/server.py b/src/docpull/server.py index c987c6d..035f29d 100644 --- a/src/docpull/server.py +++ b/src/docpull/server.py @@ -127,10 +127,13 @@ def create_serve_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--search-backend", - choices=("searxng", "marginalia"), + choices=("searxng", "marginalia", "index"), help="With --compat, serve /search routes through this backend", ) parser.add_argument("--searxng-url", help="With --search-backend searxng, the SearXNG base URL") + parser.add_argument( + "--index-dir", type=Path, help="With --search-backend index, the installed shard directory" + ) parser.add_argument( "--api-key-env", metavar="NAME", @@ -206,6 +209,7 @@ async def search( exclude_domains=exclude, include_content=content, searxng_url=args.searxng_url, + index_dir=args.index_dir, ) uvicorn.run(CompatASGIApp(search=search, api_key=api_key), host=args.host, port=args.port) diff --git a/src/docpull/surface.py b/src/docpull/surface.py index ebe7651..51db67d 100644 --- a/src/docpull/surface.py +++ b/src/docpull/surface.py @@ -35,7 +35,8 @@ class CliCommand: CliCommand("watch", "Sync, diff, and export one source"), CliCommand("parse", "Parse local PDF/DOCX/text files into a v3 pack"), CliCommand("render", "Render one public URL or check optional browser backends"), - CliCommand("search", "Search the web through self-hosted SearXNG or Marginalia"), + CliCommand("search", "Search the web through self-hosted SearXNG, Marginalia, or a discovery index"), + CliCommand("index", "Build, download, and list offline discovery index shards"), CliCommand("openapi-pack", "Build a local v3 pack from an OpenAPI spec"), CliCommand("feed-pack", "Build a local v3 pack from an RSS, Atom, JSON Feed, or feed page"), CliCommand("paper-pack", "Build a local v3 pack from papers and scholarly metadata"), diff --git a/src/docpull/web_search.py b/src/docpull/web_search.py index bc80240..5640919 100644 --- a/src/docpull/web_search.py +++ b/src/docpull/web_search.py @@ -1,9 +1,11 @@ -"""Web search through backends the user chooses: self-hosted SearXNG or Marginalia. - -DocPull has no search index of its own. ``search_web`` queries a backend the -caller names explicitly, never a paid provider, then optionally fetches the top -results through the local fetch pipeline so content keeps DocPull's SSRF and -robots.txt protections. +"""Web search through backends the user chooses: self-hosted SearXNG, Marginalia, or +an offline discovery index. + +``search_web`` queries a backend the caller names explicitly, never a paid +provider, then optionally fetches the top results through the local fetch +pipeline so content keeps DocPull's SSRF and robots.txt protections. The +``index`` backend searches downloaded discovery shards and sends no query +over the network. """ from __future__ import annotations @@ -14,6 +16,7 @@ import os import sys from dataclasses import asdict, dataclass, field, replace +from pathlib import Path from typing import Any from urllib.parse import urlsplit @@ -21,7 +24,7 @@ from . import __version__ -BACKENDS = ("searxng", "marginalia") +BACKENDS = ("searxng", "marginalia", "index") MAX_RESULTS = 20 SEARXNG_URL_ENV = "DOCPULL_SEARXNG_URL" MARGINALIA_KEY_ENV = "DOCPULL_MARGINALIA_KEY" @@ -67,6 +70,7 @@ async def search_web( include_content: bool = False, searxng_url: str | None = None, marginalia_key: str | None = None, + index_dir: Path | None = None, ) -> SearchResponse: """Search the web with an explicitly chosen backend.""" query = query.strip() @@ -79,6 +83,20 @@ async def search_web( include = [_domain(item) for item in include_domains or []] exclude = [_domain(item) for item in exclude_domains or []] + if backend == "index": + from .discovery_index import search_index + + hits = search_index( + query, + index_dir=index_dir, + max_results=max_results, + include_domains=include, + exclude_domains=exclude, + ) + if include_content: + hits = await _attach_content(hits) + return SearchResponse(query=query, backend=backend, hits=hits) + key = marginalia_key or os.environ.get(MARGINALIA_KEY_ENV) or "public" headers = {"User-Agent": f"docpull/{__version__} (+https://github.com/raintree-technology/docpull)"} async with aiohttp.ClientSession(timeout=_TIMEOUT, headers=headers) as session: @@ -237,7 +255,10 @@ async def _attach_content(hits: list[SearchHit]) -> list[SearchHit]: def create_search_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="docpull search", - description="Search the web through self-hosted SearXNG or Marginalia; no paid provider is used", + description=( + "Search the web through self-hosted SearXNG, Marginalia, or an offline discovery index; " + "no paid provider is used" + ), ) parser.add_argument("query") parser.add_argument("--backend", choices=BACKENDS, required=True) @@ -250,6 +271,7 @@ def create_search_parser() -> argparse.ArgumentParser: help="Fetch each result through the local pipeline and include its Markdown", ) parser.add_argument("--searxng-url", help=f"SearXNG base URL (default: ${SEARXNG_URL_ENV})") + parser.add_argument("--index-dir", type=Path, help="With --backend index, the installed shard directory") parser.add_argument("--json", action="store_true", dest="json_output") return parser @@ -266,6 +288,7 @@ def run_search_cli(argv: list[str] | None = None) -> int: exclude_domains=args.exclude_domains, include_content=args.include_content, searxng_url=args.searxng_url, + index_dir=args.index_dir, ) ) except WebSearchError as err: diff --git a/tests/test_discovery_index.py b/tests/test_discovery_index.py new file mode 100644 index 0000000..8c037ce --- /dev/null +++ b/tests/test_discovery_index.py @@ -0,0 +1,158 @@ +"""Discovery shards: build from locked runs, verified download, and offline search.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import socket +import sqlite3 +from pathlib import Path +from typing import Any + +import pytest + +from docpull import discovery_index +from docpull import project as project_module +from docpull.cli import main +from docpull.discovery_index import DiscoveryIndexError, build_shard, pull, search_index +from docpull.project import init_project, sync_project +from docpull.web_search import WebSearchError, search_web + +PAGES = { + "https://docs.alpha.example/retry": ( + "Retry policies", + "# Retry policies\n\n## Backoff\n\nSECRET BODY TEXT", + ), + "https://docs.alpha.example/auth": ("Authentication", "# Authentication\n\n## Retry on 401\n\nbody"), + "https://blog.beta.example/post": ("Release notes", "# Release notes\n\n```\n# not a heading\n```\n"), +} + + +def _project(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "proj" + init_project(name="proj", source="https://docs.alpha.example/", root=root) + + async def fetch(**kwargs: Any) -> dict[str, Any]: + records = [ + { + "document_id": f"doc_{i}", + "url": url, + "title": title, + "content": content, + "content_hash": hashlib.sha256(content.encode()).hexdigest(), + "fetched_at": "2026-10-12T00:00:00Z", + "metadata": {"docpull_project_source": kwargs["source"].name}, + } + for i, (url, (title, content)) in enumerate(PAGES.items()) + ] + return {"records": records, "errors": [], "skips": [], "stats": {}} + + monkeypatch.setattr(project_module, "_sync_source", fetch) + sync_project(root=root, run_id="proj_run") + return root + + +@pytest.fixture +def shards(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + out = tmp_path / "published" + build_shard([_project(tmp_path, monkeypatch)], out, "alpha") + return out + + +def test_build_writes_titles_and_headings_but_no_bodies(shards: Path) -> None: + manifest = json.loads((shards / "manifest.json").read_text()) + [entry] = manifest["shards"] + shard = shards / "alpha.sqlite" + assert entry["sha256"] == hashlib.sha256(shard.read_bytes()).hexdigest() + assert entry["documents"] == 3 and entry["domains"] == ["blog.beta.example", "docs.alpha.example"] + with sqlite3.connect(shard) as conn: + rows = dict(conn.execute("SELECT url, headings FROM pages").fetchall()) + assert rows["https://docs.alpha.example/retry"] == "Retry policies\nBackoff" + assert rows["https://blog.beta.example/post"] == "Release notes" + assert b"SECRET BODY TEXT" not in shard.read_bytes() + + +def _serve(monkeypatch: pytest.MonkeyPatch, files: dict[str, bytes]) -> list[str]: + fetched: list[str] = [] + + async def fake_fetch(url: str, limit: int) -> bytes: + fetched.append(url) + data = files[url] + if len(data) > limit: + raise DiscoveryIndexError("too large") + return data + + monkeypatch.setattr(discovery_index, "_fetch_bytes", fake_fetch) + return fetched + + +def test_pull_verifies_and_skips_current_shards( + shards: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + base = "https://index.example/v1/" + fetched = _serve( + monkeypatch, + { + base + "manifest.json": (shards / "manifest.json").read_bytes(), + base + "alpha.sqlite": (shards / "alpha.sqlite").read_bytes(), + }, + ) + installed = tmp_path / "installed" + first = asyncio.run(pull(base + "manifest.json", installed)) + assert first["downloaded"] == ["alpha"] and (installed / "alpha.sqlite").is_file() + second = asyncio.run(pull(base + "manifest.json", installed)) + assert second["downloaded"] == [] and second["up_to_date"] == ["alpha"] + assert fetched.count(base + "alpha.sqlite") == 1 + + with pytest.raises(DiscoveryIndexError, match="limit"): + asyncio.run(pull(base + "manifest.json", tmp_path / "small", max_bytes=10)) + with pytest.raises(DiscoveryIndexError, match="no shard named"): + asyncio.run(pull(base + "manifest.json", installed, shards=["missing"])) + + +def test_pull_rejects_tampered_shard(shards: Path, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + base = "https://index.example/" + data = bytearray((shards / "alpha.sqlite").read_bytes()) + data[-1] ^= 1 + _serve( + monkeypatch, + {base + "manifest.json": (shards / "manifest.json").read_bytes(), base + "alpha.sqlite": bytes(data)}, + ) + with pytest.raises(DiscoveryIndexError, match="sha256 mismatch"): + asyncio.run(pull(base + "manifest.json", tmp_path / "installed")) + assert not (tmp_path / "installed" / "alpha.sqlite").exists() + + +@pytest.mark.parametrize("url", ["http://index.example/manifest.json", "https://127.0.0.1/manifest.json"]) +def test_fetch_refuses_plain_http_and_private_hosts(url: str) -> None: + with pytest.raises(DiscoveryIndexError, match="Refusing"): + asyncio.run(discovery_index._fetch_bytes(url, 100)) + + +def test_index_backend_searches_offline(shards: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def no_network(*args: Any, **kwargs: Any) -> None: + pytest.fail("index search attempted network access") + + monkeypatch.setattr(socket, "create_connection", no_network) + response = asyncio.run(search_web("retry", backend="index", index_dir=shards)) + assert [hit.url for hit in response.hits] == [ + "https://docs.alpha.example/retry", + "https://docs.alpha.example/auth", + ] + assert response.hits[0].snippet == "Retry policies · Backoff" + filtered = search_index("retry release", index_dir=shards, exclude_domains=["docs.alpha.example"]) + assert [hit.url for hit in filtered] == ["https://blog.beta.example/post"] + with pytest.raises(WebSearchError, match="index pull"): + search_index("retry", index_dir=shards / "missing") + + +def test_index_cli_build_and_list( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + root = _project(tmp_path, monkeypatch) + out = tmp_path / "cli" + assert main(["index", "build", "--project", str(root), "--out", str(out), "--name", "alpha"]) == 0 + assert main(["index", "--index-dir", str(out), "list"]) == 0 + assert "alpha\t" in capsys.readouterr().out + assert main(["index", "pull"]) == 1 From 27a332de44a67a344ac975df4731408c4246d38e Mon Sep 17 00:00:00 2001 From: admin-raintree <277948009+admin-raintree@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:39:16 -0700 Subject: [PATCH 2/2] Share reproducible pack recipes through a GitHub registry docpull registry export writes a pack's public crawl sources, pinned page URLs, and per-page content hashes, never fetched content, and refuses sources that use auth. docpull add name@version downloads the recipe through the registry index with a sha256 check, adds and syncs the sources, and reports which pages still match. Registry recipes may only use https crawl sources, so a recipe cannot point add at local files or typed specs. docpull registry verify repeats the comparison later. Co-Authored-By: Claude Opus 5.5 --- docs/CHANGELOG.md | 5 + docs/cli-recipes.md | 34 ++++ docs/surface-contract.md | 1 + src/docpull/cli.py | 4 + src/docpull/project.py | 16 +- src/docpull/registry.py | 326 +++++++++++++++++++++++++++++++++++++++ src/docpull/surface.py | 1 + tests/test_registry.py | 185 ++++++++++++++++++++++ 8 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 src/docpull/registry.py create mode 100644 tests/test_registry.py diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f4e7698..c860040 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -24,6 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 are SQLite files of page URLs, titles, and headings from synced projects, with no page bodies. `pull` needs an explicit HTTPS manifest URL. It verifies each shard's sha256 and enforces a total download size limit. Searches run offline. +- Share reproducible packs as recipes. `docpull registry export` writes a pack's + public sources, pinned page URLs, and per-page content hashes, with no fetched + content. `docpull add name@version` downloads a recipe from the registry, + checks its sha256, adds and syncs the sources, and reports which pages still + match. `docpull registry verify` repeats the check after a later sync. ### Removed diff --git a/docs/cli-recipes.md b/docs/cli-recipes.md index 742c822..5753872 100644 --- a/docs/cli-recipes.md +++ b/docs/cli-recipes.md @@ -113,6 +113,40 @@ Shards install to `DOCPULL_INDEX_DIR`, or `$XDG_DATA_HOME/docpull/index` when that is unset. Pass `--index-dir` to `index`, `search`, or `serve` to use another directory. +## Shared Pack Recipes + +A recipe lists a pack's public sources, its pinned page URLs, and the content hash +of every page the publisher synced. It contains no fetched content. When you add +a recipe, DocPull fetches every page locally and reports which pages still match +the publisher's hashes. + +```bash +# Consumer: add, sync, and verify a published pack +docpull add stripe-api@2026-10 +docpull registry verify stripe-api@2026-10 --json + +# Publisher: export a recipe from a synced project +docpull registry export --name stripe-api --version 2026-10 --description "Stripe API reference" +``` + +`docpull add name@version`: +1. Downloads the recipe through the registry's `index.json`, checking the + recipe's sha256 against its index entry. +2. Adds the recipe's sources with their pinned URLs. +3. Saves the recipe under `.docpull/registry/`. +4. Runs `docpull sync`. +5. Prints how many pages match, changed, went missing, or are new. + +Live pages change, so drift is reported, not treated as an error. Use +`name@latest` for the newest version. + +Recipes may only contain HTTPS crawl sources. Export refuses sources that use +`auth`, so private content never goes into a registry. + +The registry is a public GitHub repository, read over HTTPS. Publishing a recipe +means opening a pull request that adds the recipe file and its `index.json` +entry. Set `DOCPULL_REGISTRY_URL` to use another registry. + ## Local Extract API `docpull serve --compat` accepts the request shapes of common hosted extract diff --git a/docs/surface-contract.md b/docs/surface-contract.md index 6d0a988..af91c86 100644 --- a/docs/surface-contract.md +++ b/docs/surface-contract.md @@ -44,6 +44,7 @@ DocPull targets capability alignment, not 1:1 flag parity. MCP should not mirror | Project lifecycle | `docpull init`, `add`, `install`, `deps`, `sources`, `sync`, `diff`, `status`, `history`, `review`, `release context-pack`, `watch` with explicit crawl bounds | `docpull.project` helpers | Not exposed yet | Core-aligned | | Search the web | `docpull search QUERY --backend searxng\|marginalia\|index` with domain filters and `--include-content` | `search_web`, `SearchResponse`, `SearchHit` in `docpull.web_search` | Not exposed | CLI-first | | Offline discovery index | `docpull index build\|pull\|list` | `build_shard`, `pull`, `search_index` in `docpull.discovery_index` | Not exposed | CLI-first | +| Shared pack recipes | `docpull add name@version`, `docpull registry export\|verify` | `export_recipe`, `fetch_recipe`, `install_recipe`, `verify_recipe` in `docpull.registry` | Not exposed | CLI-first | | Search cached sources | SQLite/local search helpers where applicable | `search_sqlite_documents` | `grep_docs` | Adapted | | Query locked context with budgets, version filters, deduplication, and precise citations | `docpull query` | `query_context`, `ContextQueryError` | `context_query` | Core-aligned | | Read cached source by path/range | Filesystem responsibility | Filesystem responsibility | `read_doc` | MCP-specific | diff --git a/src/docpull/cli.py b/src/docpull/cli.py index e28e710..b56741c 100644 --- a/src/docpull/cli.py +++ b/src/docpull/cli.py @@ -1687,6 +1687,10 @@ def main(argv: list[str] | None = None) -> int: from .graph import run_graph_cli return run_graph_cli(raw_argv[1:]) + if raw_argv and raw_argv[0] == "registry": + from .registry import run_registry_cli + + return run_registry_cli(raw_argv[1:]) if raw_argv and raw_argv[0] == "index": from .discovery_index import run_index_cli diff --git a/src/docpull/project.py b/src/docpull/project.py index 67521b9..baa81cc 100644 --- a/src/docpull/project.py +++ b/src/docpull/project.py @@ -407,7 +407,11 @@ def run_init_cli(argv: list[str] | None = None) -> int: def run_add_cli(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(prog="docpull add", description="Add a source to a DocPull project") - parser.add_argument("sources", nargs="+", help="HTTPS URL, bundled alias, or typed source spec") + parser.add_argument( + "sources", + nargs="+", + help="HTTPS URL, bundled alias, typed source spec, or registry pack name@version", + ) parser.add_argument("--name", help="Source name") parser.add_argument("--type", choices=SOURCE_TYPES, default="auto") parser.add_argument("--discover", action="store_true", help="Discover and store source URLs now") @@ -424,6 +428,16 @@ def run_add_cli(argv: list[str] | None = None) -> int: ) parser.add_argument("--auth-header-name", help="Header name for --auth header-env") args = parser.parse_args(argv) + if len(args.sources) == 1 and _infer_typed_source_type(args.sources[0]) is None: + from .registry import REF, run_add_recipe_cli + + if REF.fullmatch(args.sources[0].strip()): + if args.name or args.type != "auto" or args.discover or args.auth: + Console().print( + "[red]Project error:[/red] registry packs take no --name, --type, --discover, or --auth" + ) + return 1 + return run_add_recipe_cli(args.sources[0].strip()) console = Console() try: auth = _auth_ref_from_cli( diff --git a/src/docpull/registry.py b/src/docpull/registry.py new file mode 100644 index 0000000..b1c488e --- /dev/null +++ b/src/docpull/registry.py @@ -0,0 +1,326 @@ +"""Shared pack recipes: pinned public sources plus the content hashes a publisher saw. + +A recipe never carries fetched content. ``docpull add name@version`` downloads a +recipe from the registry, adds its sources, syncs them locally, and reports which +pages still match the published hashes. Pages change, so drift is reported, not +treated as an error. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urljoin, urlsplit + +from pydantic import BaseModel, Field, ValidationError, field_validator + +from . import __version__ +from .discovery_index import _fetch_bytes +from .project import ( + CRAWL_SOURCE_TYPES, + ProjectError, + ProjectSource, + _read_context_lock, + _read_jsonl, + _run_dir_for_id, + find_project_root, + load_project_config, + project_paths, + save_project_config, + sync_project, +) + +SCHEMA_VERSION = 1 +REGISTRY_URL_ENV = "DOCPULL_REGISTRY_URL" +DEFAULT_REGISTRY_URL = "https://raw.githubusercontent.com/raintree-technology/docpull-registry/main/" +MAX_INDEX_BYTES = 1_000_000 +# About 110 bytes per page, so 10 MB covers roughly 90,000 pages. +MAX_RECIPE_BYTES = 10_000_000 +PACK_NAME = r"[a-z0-9][a-z0-9-]{0,63}" +PACK_VERSION = r"[A-Za-z0-9._-]{1,32}" +REF = re.compile(rf"({PACK_NAME})@({PACK_VERSION})") + + +class RegistryError(ValueError): + """Invalid recipe, registry entry, or project state.""" + + +class RecipeSource(BaseModel): + """A public crawl source; typed specs and local paths are refused.""" + + name: str + url: str + type: str = "auto" + discovered_urls: list[str] = Field(default_factory=list) + + model_config = {"extra": "forbid"} + + @field_validator("url") + @classmethod + def _https(cls, value: str) -> str: + parsed = urlsplit(value) + if parsed.scheme != "https" or not parsed.netloc or parsed.username or parsed.password: + raise ValueError("recipe source URLs must be absolute https URLs without credentials") + return value + + @field_validator("type") + @classmethod + def _crawl_type(cls, value: str) -> str: + if value not in CRAWL_SOURCE_TYPES: + raise ValueError(f"recipe sources must use a crawl type: {', '.join(CRAWL_SOURCE_TYPES)}") + return value + + def project_source(self) -> ProjectSource: + # Pinned URLs make the page set reproducible, so they replace live discovery. + return ProjectSource( + name=self.name, + url=self.url, + type=self.type, # type: ignore[arg-type] + discover=not self.discovered_urls, + discovered_urls=self.discovered_urls, + ) + + +class Recipe(BaseModel): + schema_version: int + name: str = Field(pattern=rf"^{PACK_NAME}$") + version: str = Field(pattern=rf"^{PACK_VERSION}$") + description: str = "" + docpull_version: str = "" + sources: list[RecipeSource] = Field(min_length=1) + expected: dict[str, dict[str, str]] + + model_config = {"extra": "forbid"} + + @field_validator("schema_version") + @classmethod + def _schema(cls, value: int) -> int: + if value != SCHEMA_VERSION: + raise ValueError(f"unsupported recipe schema_version {value}") + return value + + def canonical(self) -> bytes: + return (json.dumps(self.model_dump(mode="json"), indent=2, sort_keys=True) + "\n").encode() + + +def _locked_pages(root: Path) -> dict[str, dict[str, str]]: + """Map each source to {url: content_hash} for the locked run.""" + paths = project_paths(root) + lock = _read_context_lock(paths.context_lock) + if not lock or not isinstance(lock.get("run_id"), str): + raise RegistryError(f"No locked run in {root}; run `docpull sync` first") + _, run_dir = _run_dir_for_id(paths, lock["run_id"]) + pages: dict[str, dict[str, str]] = {} + for record in _read_jsonl(run_dir / "documents.jsonl"): + metadata = record.get("metadata") + source = metadata.get("docpull_project_source") if isinstance(metadata, dict) else None + url, digest = record.get("url"), record.get("content_hash") + if isinstance(source, str) and isinstance(url, str) and isinstance(digest, str): + pages.setdefault(source, {})[url] = digest + return pages + + +def export_recipe(root: Path, name: str, version: str, description: str = "") -> Recipe: + """Build a recipe from a synced project's sources and locked page hashes.""" + try: + root = find_project_root(root) + config = load_project_config(root) + pages = _locked_pages(root) + except ProjectError as err: + raise RegistryError(str(err)) from err + private = [source.name for source in config.sources if source.auth] + if private: + raise RegistryError(f"Sources with auth cannot be shared: {', '.join(private)}") + try: + return Recipe( + schema_version=SCHEMA_VERSION, + name=name, + version=version, + description=description, + docpull_version=__version__, + sources=[ + RecipeSource(name=s.name, url=s.url, type=s.type, discovered_urls=s.discovered_urls) + for s in config.sources + ], + expected={ + source.name: dict(sorted(pages.get(source.name, {}).items())) for source in config.sources + }, + ) + except ValidationError as err: + raise RegistryError(f"Project cannot be exported as a recipe: {err}") from err + + +def registry_url() -> str: + url = os.environ.get(REGISTRY_URL_ENV) or DEFAULT_REGISTRY_URL + return url if url.endswith("/") else url + "/" + + +async def fetch_recipe(ref: str, base_url: str | None = None) -> Recipe: + """Resolve ``name@version`` through the registry index and verify the recipe's sha256.""" + match = REF.fullmatch(ref) + if not match: + raise RegistryError(f"Pack references look like name@version: {ref!r}") + name, version = match.groups() + base = base_url or registry_url() + try: + index = json.loads(await _fetch_bytes(urljoin(base, "index.json"), MAX_INDEX_BYTES)) + except json.JSONDecodeError as err: + raise RegistryError("Registry index is not valid JSON") from err + if not isinstance(index, dict) or index.get("schema_version") != SCHEMA_VERSION: + raise RegistryError(f"Unsupported registry index; expected schema_version {SCHEMA_VERSION}") + pack = (index.get("packs") or {}).get(name) + if not isinstance(pack, dict): + raise RegistryError(f"Registry has no pack named {name}") + if version == "latest": + version = str(pack.get("latest") or "") + entry = (pack.get("versions") or {}).get(version) + if not isinstance(entry, dict) or not isinstance(entry.get("path"), str): + raise RegistryError(f"Registry has no version {version or 'latest'} of {name}") + if not re.fullmatch(r"[0-9a-f]{64}", str(entry.get("sha256"))): + raise RegistryError(f"Registry entry for {name}@{version} has no valid sha256") + raw = await _fetch_bytes(urljoin(base, entry["path"]), MAX_RECIPE_BYTES) + if hashlib.sha256(raw).hexdigest() != entry["sha256"]: + raise RegistryError(f"sha256 mismatch for {name}@{version}; refusing the recipe") + try: + recipe = Recipe.model_validate_json(raw) + except ValidationError as err: + raise RegistryError(f"Invalid recipe {name}@{version}: {err}") from err + if (recipe.name, recipe.version) != (name, version): + raise RegistryError(f"Registry served {recipe.name}@{recipe.version} for {name}@{version}") + return recipe + + +def _recipe_path(root: Path, name: str, version: str) -> Path: + return project_paths(root).state / "registry" / f"{name}@{version}.json" + + +def install_recipe(recipe: Recipe, root: Path | None = None) -> dict[str, Any]: + """Add a recipe's sources, sync them, and return the verification report.""" + try: + project_root = find_project_root(root or Path.cwd()) + config = load_project_config(project_root) + sources = [source.project_source() for source in recipe.sources] + except (ProjectError, ValidationError) as err: + raise RegistryError(str(err)) from err + names = {source.name for source in config.sources} + urls = {source.url for source in config.sources} + clashes = [s.name for s in sources if s.name in names or s.url in urls] + if clashes: + raise RegistryError(f"Project already has sources named or located like: {', '.join(clashes)}") + save_project_config(project_root, config.model_copy(update={"sources": [*config.sources, *sources]})) + target = _recipe_path(project_root, recipe.name, recipe.version) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(recipe.canonical()) + try: + sync_project(root=project_root) + except ProjectError as err: + raise RegistryError(f"Sources added, but sync failed: {err}") from err + return verify_recipe(project_root, f"{recipe.name}@{recipe.version}") + + +def verify_recipe(root: Path, ref: str) -> dict[str, Any]: + """Compare the locked run's pages for each recipe source with the published hashes.""" + match = REF.fullmatch(ref) + if not match: + raise RegistryError(f"Pack references look like name@version: {ref!r}") + try: + project_root = find_project_root(root) + path = _recipe_path(project_root, *match.groups()) + recipe = Recipe.model_validate_json(path.read_bytes()) + pages = _locked_pages(project_root) + except FileNotFoundError as err: + raise RegistryError(f"{ref} is not installed in this project; run `docpull add {ref}`") from err + except (ProjectError, ValidationError) as err: + raise RegistryError(str(err)) from err + report: dict[str, Any] = {"pack": ref, "sources": {}} + totals = {"expected": 0, "matched": 0, "changed": 0, "missing": 0, "new": 0} + for name, expected in recipe.expected.items(): + got = pages.get(name, {}) + counts = { + "expected": len(expected), + "matched": sum(1 for url, digest in expected.items() if got.get(url) == digest), + "changed": sum(1 for url, digest in expected.items() if url in got and got[url] != digest), + "missing": sum(1 for url in expected if url not in got), + "new": sum(1 for url in got if url not in expected), + } + report["sources"][name] = counts + for key, value in counts.items(): + totals[key] += value + report.update(totals) + report["match_ratio"] = round(totals["matched"] / totals["expected"], 4) if totals["expected"] else 1.0 + return report + + +def _print_report(report: dict[str, Any]) -> None: + print( + f"{report['pack']}: {report['matched']}/{report['expected']} pages match the published hashes " + f"({report['changed']} changed, {report['missing']} missing, {report['new']} new)" + ) + + +def run_add_recipe_cli(ref: str) -> int: + try: + report = install_recipe(asyncio.run(fetch_recipe(ref))) + except (RegistryError, ValueError) as err: + print(f"docpull add: {err}", file=sys.stderr) + return 1 + _print_report(report) + return 0 + + +def run_registry_cli(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="docpull registry", description=__doc__.splitlines()[0]) + subparsers = parser.add_subparsers(dest="command", required=True) + export = subparsers.add_parser("export", help="Write a recipe from this synced project") + export.add_argument("--name", required=True) + export.add_argument("--version", required=True) + export.add_argument("--description", default="") + export.add_argument("--out", type=Path, help="Recipe file (default: @.json)") + verify = subparsers.add_parser("verify", help="Compare the latest sync with an installed recipe") + verify.add_argument("ref", help="name@version") + verify.add_argument("--json", action="store_true", dest="json_output") + args = parser.parse_args(argv) + try: + if args.command == "export": + recipe = export_recipe(Path.cwd(), args.name, args.version, args.description) + out = args.out or Path(f"{args.name}@{args.version}.json") + data = recipe.canonical() + out.write_bytes(data) + pages = sum(len(pages) for pages in recipe.expected.values()) + print(f"Wrote {out}: {len(recipe.sources)} source(s), {pages} pages") + print(f"sha256 {hashlib.sha256(data).hexdigest()}") + unpinned = [s.name for s in recipe.sources if not s.discovered_urls] + if unpinned: + print( + f"warning: {', '.join(unpinned)} has no pinned URLs; run `docpull add URL --discover` " + "or sync with discovery so consumers fetch the same pages", + file=sys.stderr, + ) + else: + report = verify_recipe(Path.cwd(), args.ref) + if args.json_output: + print(json.dumps(report, indent=2, sort_keys=True)) + else: + _print_report(report) + except RegistryError as err: + print(f"docpull registry: {err}", file=sys.stderr) + return 1 + return 0 + + +__all__ = [ + "Recipe", + "RegistryError", + "export_recipe", + "fetch_recipe", + "install_recipe", + "run_registry_cli", + "verify_recipe", +] diff --git a/src/docpull/surface.py b/src/docpull/surface.py index 51db67d..c09bc3b 100644 --- a/src/docpull/surface.py +++ b/src/docpull/surface.py @@ -37,6 +37,7 @@ class CliCommand: CliCommand("render", "Render one public URL or check optional browser backends"), CliCommand("search", "Search the web through self-hosted SearXNG, Marginalia, or a discovery index"), CliCommand("index", "Build, download, and list offline discovery index shards"), + CliCommand("registry", "Export shareable pack recipes and verify installed ones"), CliCommand("openapi-pack", "Build a local v3 pack from an OpenAPI spec"), CliCommand("feed-pack", "Build a local v3 pack from an RSS, Atom, JSON Feed, or feed page"), CliCommand("paper-pack", "Build a local v3 pack from papers and scholarly metadata"), diff --git a/tests/test_registry.py b/tests/test_registry.py new file mode 100644 index 0000000..8f25c1f --- /dev/null +++ b/tests/test_registry.py @@ -0,0 +1,185 @@ +"""Pack recipes: export from a synced project, verified fetch, add, and drift reports.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from docpull import discovery_index, registry +from docpull import project as project_module +from docpull.cli import main +from docpull.project import init_project, load_project_config, sync_project +from docpull.registry import RegistryError, export_recipe, fetch_recipe, install_recipe, verify_recipe + +PAGES = { + "https://docs.alpha.example/retry": "# Retry\n\nWait 2 seconds.", + "https://docs.alpha.example/auth": "# Auth\n\nUse a token.", +} +BASE = "https://registry.example/" + + +def _fake_sync(monkeypatch: pytest.MonkeyPatch, pages: dict[str, str]) -> None: + async def fetch(**kwargs: Any) -> dict[str, Any]: + source = kwargs["source"] + records = [ + { + "document_id": f"doc_{i}", + "url": url, + "title": url, + "content": content, + "content_hash": hashlib.sha256(content.encode()).hexdigest(), + "metadata": {"docpull_project_source": source.name}, + } + for i, (url, content) in enumerate(pages.items()) + if url.startswith(source.url.rstrip("/")) + ] + return {"records": records, "errors": [], "skips": [], "stats": {}} + + monkeypatch.setattr(project_module, "_sync_source", fetch) + + +def _publisher(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + root = tmp_path / "publisher" + init_project(name="publisher", source="https://docs.alpha.example/", root=root) + config = load_project_config(root) + pinned = config.sources[0].model_copy(update={"discovered_urls": sorted(PAGES)}) + project_module.save_project_config(root, config.model_copy(update={"sources": [pinned]})) + _fake_sync(monkeypatch, PAGES) + sync_project(root=root, run_id="publisher_run") + return root + + +def _serve(monkeypatch: pytest.MonkeyPatch, recipe_bytes: bytes, sha256: str | None = None) -> None: + index = { + "schema_version": 1, + "packs": { + "alpha-docs": { + "latest": "2026-10", + "versions": { + "2026-10": { + "path": "packs/alpha-docs/2026-10.json", + "sha256": sha256 or hashlib.sha256(recipe_bytes).hexdigest(), + } + }, + } + }, + } + files = { + BASE + "index.json": json.dumps(index).encode(), + BASE + "packs/alpha-docs/2026-10.json": recipe_bytes, + } + + async def fake_fetch(url: str, limit: int) -> bytes: + return files[url] + + monkeypatch.setattr(registry, "_fetch_bytes", fake_fetch) + monkeypatch.setenv(registry.REGISTRY_URL_ENV, BASE) + + +def test_export_records_pinned_sources_and_page_hashes( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + recipe = export_recipe(_publisher(tmp_path, monkeypatch), "alpha-docs", "2026-10", "Alpha docs") + [source] = recipe.sources + assert source.discovered_urls == sorted(PAGES) + assert recipe.expected[source.name] == { + url: hashlib.sha256(content.encode()).hexdigest() for url, content in sorted(PAGES.items()) + } + assert b"Wait 2 seconds" not in recipe.canonical() + with pytest.raises(RegistryError, match="name"): + export_recipe(tmp_path / "publisher", "Bad Name", "1") + + +def test_export_refuses_private_sources(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = _publisher(tmp_path, monkeypatch) + config = load_project_config(root) + private = config.sources[0].model_copy( + update={"auth": project_module.ProjectSourceAuth(type="bearer_env", env="TOKEN")} + ) + project_module.save_project_config(root, config.model_copy(update={"sources": [private]})) + with pytest.raises(RegistryError, match="auth"): + export_recipe(root, "alpha-docs", "2026-10") + + +def test_fetch_verifies_sha256_and_resolves_latest(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + data = export_recipe(_publisher(tmp_path, monkeypatch), "alpha-docs", "2026-10").canonical() + _serve(monkeypatch, data) + assert asyncio.run(fetch_recipe("alpha-docs@latest")).version == "2026-10" + with pytest.raises(RegistryError, match="no version 2027"): + asyncio.run(fetch_recipe("alpha-docs@2027")) + with pytest.raises(RegistryError, match="no pack named"): + asyncio.run(fetch_recipe("beta@1")) + _serve(monkeypatch, data, sha256="0" * 64) + with pytest.raises(RegistryError, match="sha256 mismatch"): + asyncio.run(fetch_recipe("alpha-docs@2026-10")) + + +@pytest.mark.parametrize( + "source", + [ + {"name": "local", "url": "./secrets.sqlite", "type": "auto"}, + {"name": "typed", "url": "https://github.com/a/b", "type": "repo"}, + {"name": "plain", "url": "http://docs.example/", "type": "auto"}, + ], +) +def test_registry_recipes_only_allow_public_crawl_sources( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: dict[str, str] +) -> None: + payload = { + "schema_version": 1, + "name": "alpha-docs", + "version": "2026-10", + "sources": [source], + "expected": {}, + } + _serve(monkeypatch, json.dumps(payload).encode()) + with pytest.raises(RegistryError, match="Invalid recipe"): + asyncio.run(fetch_recipe("alpha-docs@2026-10")) + + +def test_add_installs_syncs_and_reports_drift( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + data = export_recipe(_publisher(tmp_path, monkeypatch), "alpha-docs", "2026-10").canonical() + _serve(monkeypatch, data) + consumer = tmp_path / "consumer" + init_project(name="consumer", root=consumer) + monkeypatch.chdir(consumer) + changed = {**PAGES, "https://docs.alpha.example/auth": "# Auth\n\nUse an API key."} + _fake_sync(monkeypatch, changed) + + assert main(["add", "alpha-docs@2026-10"]) == 0 + assert "1/2 pages match" in capsys.readouterr().out + [source] = load_project_config(consumer).sources + assert source.discover is False and source.discovered_urls == sorted(PAGES) + report = verify_recipe(consumer, "alpha-docs@2026-10") + assert (report["matched"], report["changed"], report["missing"], report["new"]) == (1, 1, 0, 0) + assert report["match_ratio"] == 0.5 + + with pytest.raises(RegistryError, match="already has sources"): + install_recipe(asyncio.run(fetch_recipe("alpha-docs@2026-10")), consumer) + assert main(["registry", "verify", "alpha-docs@2026-10", "--json"]) == 0 + assert json.loads(capsys.readouterr().out)["matched"] == 1 + + +def test_add_still_resolves_aliases_urls_and_typed_specs( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def no_registry(*args: Any, **kwargs: Any) -> None: + pytest.fail("non-registry source reached the registry") + + monkeypatch.setattr(discovery_index, "_fetch_bytes", no_registry) + monkeypatch.setattr(registry, "_fetch_bytes", no_registry) + root = tmp_path / "plain" + init_project(name="plain", root=root) + monkeypatch.chdir(root) + assert main(["add", "react"]) == 0 + assert main(["add", "https://docs.gamma.example/"]) == 0 + assert main(["add", "npm:react@18.3.1"]) == 0 + assert main(["add", "owner/repo@v1"]) == 0 + assert len(load_project_config(root).sources) == 4