wasimxyz · Oct 9, 2026 · Oct 9, 2026
diff --git a/.github/workflows/docs-lint.yml b/.github/workflows/docs-lint.yml
@@ -9,6 +9,7 @@ on:
       - .github/workflows/docs-lint.yml
       - docs/**
       - web/lib/docs.ts
+      - web/lib/mcp/docs/docs-bundle.snapshot.json
       - web/content/changelog/**
       - web/microfrontends.json
       - Makefile
@@ -20,6 +21,7 @@ on:
       - .github/workflows/docs-lint.yml
       - docs/**
       - web/lib/docs.ts
+      - web/lib/mcp/docs/docs-bundle.snapshot.json
       - web/content/changelog/**
       - web/microfrontends.json
       - Makefile
@@ -55,3 +57,15 @@ jobs:
       # the site to docs/developer, and relative links in docs/developer.
       - name: Check links
         run: npm run check:links
+
+      # The MCP server answers questions from a bundled copy of these pages.
+      # Regenerate it and fail if the committed copy differs.
+      - name: Check docs bundle for the MCP server is current
+        run: |
+          npm run docs-bundle:generate
+          bundle="../../web/lib/mcp/docs/docs-bundle.snapshot.json"
+          if [ -n "$(git status --porcelain -- $bundle)" ]; then
+            echo "::error::The docs bundle for the MCP server is out of date. Run 'make docs-catalogs' and commit the result."
+            git --no-pager diff --stat -- $bundle
+            exit 1
+          fi
diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml
@@ -82,7 +82,7 @@ jobs:
           make py-watcher-cli-catalog
           snapshot=docs/site/src/lib/cli-catalog.snapshot.json
           if [ -n "$(git status --porcelain -- $snapshot)" ]; then
-            echo "::error::The docs watcher CLI snapshot is out of date. Run 'make py-watcher-cli-catalog' and commit the result."
+            echo "::error::The docs watcher CLI snapshot is out of date. Run 'make py-watcher-cli-catalog docs-bundle-generate' and commit both files."
             git --no-pager diff --stat -- $snapshot
             exit 1
           fi
diff --git a/AGENTS.md b/AGENTS.md
@@ -23,7 +23,7 @@ The docs site deploys from the `production` branch, so a docs change goes live w
 
 ### Watcher version
 
-Shipped watcher changes (`watcher/src/`) need a version bump in the same branch. Follow `watcher/AGENTS.md`: bump `[project].version` in `watcher/pyproject.toml` once if it still matches the merge-base, then run `uv lock`, then run `make py-watcher-cli-catalog` and commit the updated `docs/site/src/lib/cli-catalog.snapshot.json`, which records the version. Do not tag the release from the feature branch.
+Shipped watcher changes (`watcher/src/`) need a version bump in the same branch. Follow `watcher/AGENTS.md`: bump `[project].version` in `watcher/pyproject.toml` once if it still matches the merge-base, then run `uv lock`, then run `make py-watcher-cli-catalog docs-bundle-generate` and commit the updated `docs/site/src/lib/cli-catalog.snapshot.json`, which records the version, and `web/lib/mcp/docs/docs-bundle.snapshot.json`, which embeds the CLI reference page. Do not tag the release from the feature branch.
 
 ### Changelog
 
@@ -38,11 +38,15 @@ Do not add an entry for tests, docs, CI, dependency updates, or infrastructure.
 The API, MCP, and [Watcher CLI](https://datahub.arcadiascience.com/docs/cli-reference) pages on the docs site render from three JSON files committed in `docs/site/src/lib/`: `openapi.snapshot.json`, `mcp-catalog.snapshot.json`, and `cli-catalog.snapshot.json`. They are generated from the code and never edited by hand.
 
 1. Change the REST API (`web/lib/api/openapi/`), the MCP tools, prompts, or resources (`web/lib/mcp/`), or the watcher CLI help or options (`watcher/src/data_hub_watcher/cli.py`).
-2. Run `make docs-catalogs`. To refresh only the watcher CLI file, run `make py-watcher-cli-catalog`.
+2. Run `make docs-catalogs`. To refresh only the watcher CLI file, run `make py-watcher-cli-catalog docs-bundle-generate`, because the [docs bundle for the MCP server](#docs-bundle-for-the-mcp-server) embeds the CLI reference page.
 3. Commit the changed snapshot files in the same branch.
 
 CI regenerates each file and fails when the committed copy differs.
 
+### Docs bundle for the MCP server
+
+The MCP server answers questions about Data Hub from a bundled copy of the docs pages, `web/lib/mcp/docs/docs-bundle.snapshot.json`. It is generated from `docs/site/content/docs/` (and the watcher CLI snapshot) and never edited by hand. After changing a docs page or the watcher CLI snapshot, including a watcher version bump, run `make docs-catalogs` and commit the result in the same branch. CI regenerates it and fails when the committed copy differs. See `docs/developer/mcp-docs.md`.
+
 ## Cursor Cloud specific instructions
 
 Data Hub is a multi-component repo (see `README.md`). The component you can run end-to-end locally with zero external credentials is the **Next.js web app + REST API + PostgreSQL** (`web/`). The `lambda/`, `watcher/`, and `packages/shared/` Python packages are exercised via tests and a local S3 mirror — no real AWS is needed for local work.

diff --git a/Makefile b/Makefile
@@ -49,7 +49,8 @@ py-check-watcher-version:
 # Regenerate the watcher CLI catalog the docs site renders its CLI reference
 # from. The snapshot is committed; the Python test workflow fails when it is
 # out of date. Run this after changing watcher/src/data_hub_watcher/cli.py or
-# bumping the watcher version.
+# bumping the watcher version, then `make docs-bundle-generate`, because the
+# MCP docs bundle embeds the CLI reference page.
 .PHONY: py-watcher-cli-catalog
 py-watcher-cli-catalog:
 	uv run python -m data_hub_watcher.cli_catalog docs/site/src/lib/cli-catalog.snapshot.json
@@ -116,14 +117,25 @@ mcp-catalog-generate:
 docs-dev:
 	cd docs/site && npm run dev
 
+# Bundle the docs pages into web/ so the MCP server can answer questions about
+# Data Hub from the docs that ship with each deployment. Run after changing a
+# docs page or the watcher CLI snapshot, which the bundle embeds. CI fails when
+# the committed bundle is stale.
+.PHONY: docs-bundle-generate
+docs-bundle-generate:
+	cd docs/site && npm run docs-bundle:generate
+
 # Regenerate the three snapshots the docs site renders its API, MCP, and
-# watcher CLI reference pages from. Run after changing the REST API, the MCP
-# tools, or the watcher CLI. CI fails when a committed snapshot is stale.
+# watcher CLI reference pages from, then the docs bundle for the MCP server,
+# which embeds the CLI snapshot. Run after changing the REST API, the MCP
+# tools, the watcher CLI, or a docs page. CI fails when a committed copy is
+# stale.
 .PHONY: docs-catalogs
 docs-catalogs:
 	make openapi-generate
 	make mcp-catalog-generate
 	make py-watcher-cli-catalog
+	make docs-bundle-generate
 
 .PHONY: docs-format
 docs-format:

diff --git a/docs/developer/README.md b/docs/developer/README.md
@@ -13,6 +13,7 @@ the docs site does not render them.
 - [Local development](local-development.md) — zero-credential dev workflow for the web app + API + database (no watcher / Lambda needed)
 - [Architecture](architecture.md) — system overview, data flow, and design decisions
 - [MCP Apps](mcp-apps.md) — run-report View: host render flow, the three app-only tools, the shared data source, `mcp-apps:build`, and the gitignored HTML artifact
+- [Docs over MCP](mcp-docs.md) — how the `search_docs` and `read_doc` tools get the docs, how search works, and what to update when docs change
 - [Testing](testing.md) — per-package test frameworks, the shared test-server fixture, S3 mocking
 - [Watcher](watcher.md) — CLI commands, configuration, run detection, upload modes
 - [Lambda](lambda.md) — processing pipeline, supported instruments, adding new instruments

diff --git a/docs/developer/ci-and-deployment.md b/docs/developer/ci-and-deployment.md
@@ -15,7 +15,7 @@ Five workflows run on pushes to `staging`/`production` and on pull requests targ
 - Starts a Postgres 17 service container.
 - Installs both Node.js 24 and Python packages.
 - `make py-test` — runs all pytest tests (unit and integration). Integration tests build and start a real Next.js production server, seed a test database, and exercise the Lambda and watcher against the live API.
-- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot.
+- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot. The MCP docs bundle embeds the same page, so run `make py-watcher-cli-catalog docs-bundle-generate` and commit both files, or `docs-lint.yml` fails next.
 
 ### TypeScript lint and typecheck (`typescript-lint.yml`)
 
@@ -26,12 +26,13 @@ Five workflows run on pushes to `staging`/`production` and on pull requests targ
 
 ### Docs lint, typecheck, and link check (`docs-lint.yml`)
 
-Runs when `docs/**`, `web/lib/docs.ts`, `web/content/changelog/**`, or `web/microfrontends.json` changes. It works in `docs/site` with Node.js 24:
+Runs when `docs/**`, `web/lib/docs.ts`, `web/lib/mcp/docs/docs-bundle.snapshot.json`, `web/content/changelog/**`, or `web/microfrontends.json` changes. It works in `docs/site` with Node.js 24:
 
 1. Install dependencies with `npm ci`.
 2. `npm run lint:check` — Biome (via Ultracite), combined formatter + linter check.
 3. `npm run typecheck` — generates page types (which also fails on bad MDX) and runs the TypeScript compiler.
 4. `npm run check:links` — checks that the page names in `web/lib/docs.ts`, the `docs:` links in changelog entries, the GitHub links from the site to `docs/developer`, and the relative links inside `docs/developer` all point at something that exists. It does not check `#heading` anchors.
+5. `npm run docs-bundle:generate`, then fails if `web/lib/mcp/docs/docs-bundle.snapshot.json` changed. The MCP server answers questions about Data Hub from that bundled copy of the docs pages (see [Docs over MCP](mcp-docs.md)). Run `make docs-catalogs` and commit the result to fix a failure.
 
 ### TypeScript tests (`typescript-test.yml`)
 

diff --git a/docs/developer/mcp-docs.md b/docs/developer/mcp-docs.md
@@ -0,0 +1,57 @@
+# Docs over MCP
+
+An assistant connected to Data Hub's MCP server can answer questions about Data Hub itself. Two tools, `search_docs` and `read_doc`, and a `datahub://docs/{page}` resource template serve the docs pages and the changelog. This page explains how they get their content, how search works, and what to update when you change them.
+
+## Where the content comes from
+
+Each deployment answers from the docs that match its own code, so the docs ship inside the web app instead of being fetched from the published site.
+
+```text
+docs/site/content/docs/*.mdx
+        │  make docs-catalogs  (docs/site/scripts/generate-docs-bundle.ts)
+        ▼
+web/lib/mcp/docs/docs-bundle.snapshot.json   (committed, never edited by hand)
+        │
+        ├── web/lib/mcp/docs/bundle.ts    pages + changelog, split into sections
+        ├── web/lib/mcp/docs/search.ts    MiniSearch index, built once per server
+        └── web/lib/mcp/tools/docs.ts     search_docs and read_doc
+```
+
+- The generator uses the same `getLLMText` function as `/docs/<page>.md`, so the Markdown an agent reads matches what the site serves.
+- It includes the hand-written pages, such as `cli-reference`, `mcp`, and `api` (the API overview). It leaves out the generated API endpoint pages and the MCP tool, prompt, and resource catalogs, because the client already receives the catalog when it connects.
+- The `cli-reference` page includes the watcher version, so a watcher version bump changes the bundle. Run `make py-watcher-cli-catalog docs-bundle-generate` and commit both files.
+- The changelog is read from `web/content/changelog/` at runtime, the same files the in-app changelog uses. Deployments get those files through `outputFileTracingIncludes` in `web/next.config.mjs`.
+- `docs-lint.yml` regenerates the file and fails when the committed copy differs. After changing a docs page, run `make docs-catalogs` and commit the result.
+- A change to the bundle is a change under `web/`, so the web app's Vercel project rebuilds for docs-only changes. That is intended: the deployed MCP server has to carry the new text.
+
+## How the agent finds the right page
+
+No tool returns all the docs at once. The flow is search, then read.
+
+1. `search_docs(query)` returns up to five matching sections, with at most two from one page. Each has a page ID, a section ID, a short excerpt, and a link to cite.
+2. `read_doc(page, section?)` returns one page, or one section of it, as Markdown. It also lists the page's section IDs. It accepts a docs URL or a `/docs/...` path as well as a page ID. For an unknown page ID, the error suggests the closest page IDs, allowing for typos and other forms of a word.
+3. When no section matches most of the question, `search_docs` adds a hint: the results may not answer it, so the agent should say the docs don't cover the question and offer `send_feedback` about the gap. The hint also names any words of the question that appear nowhere in the docs.
+4. When nothing matches at all, `search_docs` returns every docs page with its description, and the same hint to offer `send_feedback`.
+
+Search ranks sections with [MiniSearch](https://lucaong.github.io/minisearch/). Headings and page titles count more than body text. Common question words and the product name are dropped, plurals are folded together, and long words tolerate a typo. Changelog entries are weighted lower so a short entry does not outrank the page that explains the feature. There are no embeddings, so there is no extra service or API key.
+
+A section matches when it contains any one word of the question, so most questions return results even when the docs don't cover them. A section counts as an answer when it contains at least two thirds of the question's words, rounded up, and every word of the question appears somewhere in the docs. That rule decides when the hint in step 3 appears. Some uncovered questions still pass it, for example when every word appears on an unrelated page, so the `search_docs` description also tells the agent to judge the results itself.
+
+Page IDs are the page's path under `docs/site/content/docs/` (`manage-tokens`, `mcp`), and changelog entries are `changelog/<file name>`. Section IDs are the `[#id]` after a heading, or the generated slug.
+
+## Server instructions
+
+`web/lib/mcp/instructions.ts` is sent to every client at connect time. Claude Code keeps only the first 2 KB, so a unit test (`tests/unit/mcp-instructions.test.ts`) fails above 2,048 bytes. Put the most important points first.
+
+## Usage data
+
+`mcp_docs_search` records how many sections answer the question by the rule above, in the same buckets as the in-app search, and `mcp_docs_read` records the page ID. The query text is never recorded. A rising share of searches in the `0` bucket shows where the docs have gaps.
+
+## Tests
+
+- `tests/unit/mcp-docs-search.test.ts` holds real questions, each with the page that should answer it. The page must be in the top three results, and the search must count at least one answer. It also holds questions the docs don't answer, which must get no answers. Add a question when someone reports a search that missed.
+- `tests/unit/mcp-docs-markdown.test.ts` covers section splitting, page references, and page ID suggestions.
+- `tests/mcp/mcp-protocol.test.ts` covers the tools and the resource template over the in-memory transport.
+- `tests/integration/mcp.test.ts` calls both tools against the production build, which shows the docs bundle is compiled into the server. It can't show that the changelog files reach a deployment, because `next start` reads them from the source tree.
+
+Adding the two tools took the tool count from 37 to 39. Pinned counts in `tests/mcp/mcp-catalog.test.ts`, `tests/mcp/mcp-protocol.test.ts`, and `tests/integration/mcp.test.ts` show that change in review.
diff --git a/docs/site/AGENTS.md b/docs/site/AGENTS.md
@@ -30,15 +30,26 @@ Run every command below from `docs/site/`, or use the `make docs-*` targets from
 ## Watcher CLI catalog
 
 - [`content/docs/cli-reference.mdx`](content/docs/cli-reference.mdx) renders `<WatcherCliCatalog />` from the committed snapshot at `src/lib/cli-catalog.snapshot.json`.
-- Regenerate it with `make py-watcher-cli-catalog` (or `make docs-catalogs`, which also refreshes the other two snapshots) and commit the result. CI fails when it is out of date. A watcher version bump changes it too, because the snapshot records the version.
+- Regenerate it with `make py-watcher-cli-catalog docs-bundle-generate` (or `make docs-catalogs`, which also refreshes the other two snapshots) and commit the result. CI fails when it is out of date. A watcher version bump changes it too, because the snapshot records the version. The [MCP docs bundle](#mcp-docs-bundle) embeds this page, so it changes with it.
 - Keep narrative caveats (interactive prompts, Windows-only service, examples) in the MDX; do not hand-maintain flag tables.
 - Markdown / LLM export expands `WatcherCliCatalog` via `getLLMText` (`src/lib/cli-catalog-markdown.ts`), same placeholder pattern as the MCP catalogs.
 
 The three `src/lib/*.snapshot.json` files are generated output, so Biome skips them. Never edit them by hand.
 
+## MCP docs bundle
+
+The Data Hub MCP server answers questions from a copy of these pages bundled into the web app, so each deployment answers from the docs that match its own code.
+
+- `scripts/generate-docs-bundle.ts` writes the copy to `web/lib/mcp/docs/docs-bundle.snapshot.json`, using the same `getLLMText` output as `/docs/<path>.md`. It leaves out the generated API endpoint pages and the MCP tool, prompt, and resource catalogs.
+- Regenerate it with `make docs-catalogs` (or `npm run docs-bundle:generate` here) after changing any page or the watcher CLI snapshot, and commit the result. CI fails when it is out of date. Never edit it by hand.
+- The script loads the pages through `fumadocs-mdx/node`, which needs this package to be an ES module (`"type": "module"`), and runs under `tsx`. `scripts/stub-assets-loader.mjs` stubs image imports, which Node cannot load.
+- The server returns a page in one tool result, so a page must stay under 75,000 characters (the longest today is about 18,000). A test fails above that, because Claude Code caps a tool result at 25,000 tokens. Split a page that grows past it.
+- Give every heading a stable `[#id]` when other pages link to it. The server uses those IDs as section names, and agents quote them back.
+- Search quality is covered by a list of real questions in `web/tests/unit/mcp-docs-search.test.ts`, plus questions the docs don't answer, which must come back flagged as weak matches. When a page is renamed or split, or a new page answers one of those questions, update the lists.
+
 ## AI-consumption surface
 
-This site is intentionally public and indexable, and is itself built to be read by agents: `/llms.txt` (page index), `/llms-full.txt` (full corpus dump), and `/docs/<path>.md` (any page as raw Markdown, via content negotiation on `Accept: text/markdown`). When editing docs, keep pages self-contained enough that they still make sense pulled out of the nav and read as a flat Markdown file — that's how most AI clients will actually consume them.
+This site is intentionally public and indexable, and is itself built to be read by agents: `/llms.txt` (page index), `/llms-full.txt` (full corpus dump), and `/docs/<path>.md` (any page as raw Markdown, via content negotiation on `Accept: text/markdown`). The Data Hub MCP server serves the same pages through its `search_docs` and `read_doc` tools (see [MCP docs bundle](#mcp-docs-bundle)). When editing docs, keep pages self-contained enough that they still make sense pulled out of the nav and read as a flat Markdown file — that's how most AI clients will actually consume them.
 
 ## Checks