diff --git a/.github/workflows/docs-lint.yml b/.github/workflows/docs-lint.yml index c7e4f3ab..99dec235 100644 --- a/.github/workflows/docs-lint.yml +++ b/.github/workflows/docs-lint.yml @@ -9,6 +9,7 @@ on: - .github/workflows/docs-lint.yml - docs/** - web/lib/docs.ts + - web/lib/mcp/docs/docs-bundle.snapshot.json - web/content/changelog/** - web/microfrontends.json - Makefile @@ -20,6 +21,7 @@ on: - .github/workflows/docs-lint.yml - docs/** - web/lib/docs.ts + - web/lib/mcp/docs/docs-bundle.snapshot.json - web/content/changelog/** - web/microfrontends.json - Makefile @@ -55,3 +57,15 @@ jobs: # the site to docs/developer, and relative links in docs/developer. - name: Check links run: npm run check:links + + # The MCP server answers questions from a bundled copy of these pages. + # Regenerate it and fail if the committed copy differs. + - name: Check docs bundle for the MCP server is current + run: | + npm run docs-bundle:generate + bundle="../../web/lib/mcp/docs/docs-bundle.snapshot.json" + if [ -n "$(git status --porcelain -- $bundle)" ]; then + echo "::error::The docs bundle for the MCP server is out of date. Run 'make docs-catalogs' and commit the result." + git --no-pager diff --stat -- $bundle + exit 1 + fi diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index aad4e41a..28b1d817 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -82,7 +82,7 @@ jobs: make py-watcher-cli-catalog snapshot=docs/site/src/lib/cli-catalog.snapshot.json if [ -n "$(git status --porcelain -- $snapshot)" ]; then - echo "::error::The docs watcher CLI snapshot is out of date. Run 'make py-watcher-cli-catalog' and commit the result." + echo "::error::The docs watcher CLI snapshot is out of date. Run 'make py-watcher-cli-catalog docs-bundle-generate' and commit both files." git --no-pager diff --stat -- $snapshot exit 1 fi diff --git a/AGENTS.md b/AGENTS.md index 680903e9..b304b8f0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ The docs site deploys from the `production` branch, so a docs change goes live w ### Watcher version -Shipped watcher changes (`watcher/src/`) need a version bump in the same branch. Follow `watcher/AGENTS.md`: bump `[project].version` in `watcher/pyproject.toml` once if it still matches the merge-base, then run `uv lock`, then run `make py-watcher-cli-catalog` and commit the updated `docs/site/src/lib/cli-catalog.snapshot.json`, which records the version. Do not tag the release from the feature branch. +Shipped watcher changes (`watcher/src/`) need a version bump in the same branch. Follow `watcher/AGENTS.md`: bump `[project].version` in `watcher/pyproject.toml` once if it still matches the merge-base, then run `uv lock`, then run `make py-watcher-cli-catalog docs-bundle-generate` and commit the updated `docs/site/src/lib/cli-catalog.snapshot.json`, which records the version, and `web/lib/mcp/docs/docs-bundle.snapshot.json`, which embeds the CLI reference page. Do not tag the release from the feature branch. ### Changelog @@ -38,11 +38,15 @@ Do not add an entry for tests, docs, CI, dependency updates, or infrastructure. The API, MCP, and [Watcher CLI](https://datahub.arcadiascience.com/docs/cli-reference) pages on the docs site render from three JSON files committed in `docs/site/src/lib/`: `openapi.snapshot.json`, `mcp-catalog.snapshot.json`, and `cli-catalog.snapshot.json`. They are generated from the code and never edited by hand. 1. Change the REST API (`web/lib/api/openapi/`), the MCP tools, prompts, or resources (`web/lib/mcp/`), or the watcher CLI help or options (`watcher/src/data_hub_watcher/cli.py`). -2. Run `make docs-catalogs`. To refresh only the watcher CLI file, run `make py-watcher-cli-catalog`. +2. Run `make docs-catalogs`. To refresh only the watcher CLI file, run `make py-watcher-cli-catalog docs-bundle-generate`, because the [docs bundle for the MCP server](#docs-bundle-for-the-mcp-server) embeds the CLI reference page. 3. Commit the changed snapshot files in the same branch. CI regenerates each file and fails when the committed copy differs. +### Docs bundle for the MCP server + +The MCP server answers questions about Data Hub from a bundled copy of the docs pages, `web/lib/mcp/docs/docs-bundle.snapshot.json`. It is generated from `docs/site/content/docs/` (and the watcher CLI snapshot) and never edited by hand. After changing a docs page or the watcher CLI snapshot, including a watcher version bump, run `make docs-catalogs` and commit the result in the same branch. CI regenerates it and fails when the committed copy differs. See `docs/developer/mcp-docs.md`. + ## Cursor Cloud specific instructions Data Hub is a multi-component repo (see `README.md`). The component you can run end-to-end locally with zero external credentials is the **Next.js web app + REST API + PostgreSQL** (`web/`). The `lambda/`, `watcher/`, and `packages/shared/` Python packages are exercised via tests and a local S3 mirror — no real AWS is needed for local work. diff --git a/Makefile b/Makefile index e3a7a7b6..58885531 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,8 @@ py-check-watcher-version: # Regenerate the watcher CLI catalog the docs site renders its CLI reference # from. The snapshot is committed; the Python test workflow fails when it is # out of date. Run this after changing watcher/src/data_hub_watcher/cli.py or -# bumping the watcher version. +# bumping the watcher version, then `make docs-bundle-generate`, because the +# MCP docs bundle embeds the CLI reference page. .PHONY: py-watcher-cli-catalog py-watcher-cli-catalog: uv run python -m data_hub_watcher.cli_catalog docs/site/src/lib/cli-catalog.snapshot.json @@ -116,14 +117,25 @@ mcp-catalog-generate: docs-dev: cd docs/site && npm run dev +# Bundle the docs pages into web/ so the MCP server can answer questions about +# Data Hub from the docs that ship with each deployment. Run after changing a +# docs page or the watcher CLI snapshot, which the bundle embeds. CI fails when +# the committed bundle is stale. +.PHONY: docs-bundle-generate +docs-bundle-generate: + cd docs/site && npm run docs-bundle:generate + # Regenerate the three snapshots the docs site renders its API, MCP, and -# watcher CLI reference pages from. Run after changing the REST API, the MCP -# tools, or the watcher CLI. CI fails when a committed snapshot is stale. +# watcher CLI reference pages from, then the docs bundle for the MCP server, +# which embeds the CLI snapshot. Run after changing the REST API, the MCP +# tools, the watcher CLI, or a docs page. CI fails when a committed copy is +# stale. .PHONY: docs-catalogs docs-catalogs: make openapi-generate make mcp-catalog-generate make py-watcher-cli-catalog + make docs-bundle-generate .PHONY: docs-format docs-format: diff --git a/docs/developer/README.md b/docs/developer/README.md index 2014e230..39c12b42 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -13,6 +13,7 @@ the docs site does not render them. - [Local development](local-development.md) — zero-credential dev workflow for the web app + API + database (no watcher / Lambda needed) - [Architecture](architecture.md) — system overview, data flow, and design decisions - [MCP Apps](mcp-apps.md) — run-report View: host render flow, the three app-only tools, the shared data source, `mcp-apps:build`, and the gitignored HTML artifact +- [Docs over MCP](mcp-docs.md) — how the `search_docs` and `read_doc` tools get the docs, how search works, and what to update when docs change - [Testing](testing.md) — per-package test frameworks, the shared test-server fixture, S3 mocking - [Watcher](watcher.md) — CLI commands, configuration, run detection, upload modes - [Lambda](lambda.md) — processing pipeline, supported instruments, adding new instruments diff --git a/docs/developer/ci-and-deployment.md b/docs/developer/ci-and-deployment.md index af813cb9..c74e83b2 100644 --- a/docs/developer/ci-and-deployment.md +++ b/docs/developer/ci-and-deployment.md @@ -15,7 +15,7 @@ Five workflows run on pushes to `staging`/`production` and on pull requests targ - Starts a Postgres 17 service container. - Installs both Node.js 24 and Python packages. - `make py-test` — runs all pytest tests (unit and integration). Integration tests build and start a real Next.js production server, seed a test database, and exercise the Lambda and watcher against the live API. -- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot. +- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot. The MCP docs bundle embeds the same page, so run `make py-watcher-cli-catalog docs-bundle-generate` and commit both files, or `docs-lint.yml` fails next. ### TypeScript lint and typecheck (`typescript-lint.yml`) @@ -26,12 +26,13 @@ Five workflows run on pushes to `staging`/`production` and on pull requests targ ### Docs lint, typecheck, and link check (`docs-lint.yml`) -Runs when `docs/**`, `web/lib/docs.ts`, `web/content/changelog/**`, or `web/microfrontends.json` changes. It works in `docs/site` with Node.js 24: +Runs when `docs/**`, `web/lib/docs.ts`, `web/lib/mcp/docs/docs-bundle.snapshot.json`, `web/content/changelog/**`, or `web/microfrontends.json` changes. It works in `docs/site` with Node.js 24: 1. Install dependencies with `npm ci`. 2. `npm run lint:check` — Biome (via Ultracite), combined formatter + linter check. 3. `npm run typecheck` — generates page types (which also fails on bad MDX) and runs the TypeScript compiler. 4. `npm run check:links` — checks that the page names in `web/lib/docs.ts`, the `docs:` links in changelog entries, the GitHub links from the site to `docs/developer`, and the relative links inside `docs/developer` all point at something that exists. It does not check `#heading` anchors. +5. `npm run docs-bundle:generate`, then fails if `web/lib/mcp/docs/docs-bundle.snapshot.json` changed. The MCP server answers questions about Data Hub from that bundled copy of the docs pages (see [Docs over MCP](mcp-docs.md)). Run `make docs-catalogs` and commit the result to fix a failure. ### TypeScript tests (`typescript-test.yml`) diff --git a/docs/developer/mcp-docs.md b/docs/developer/mcp-docs.md new file mode 100644 index 00000000..bcb8a6a2 --- /dev/null +++ b/docs/developer/mcp-docs.md @@ -0,0 +1,57 @@ +# Docs over MCP + +An assistant connected to Data Hub's MCP server can answer questions about Data Hub itself. Two tools, `search_docs` and `read_doc`, and a `datahub://docs/{page}` resource template serve the docs pages and the changelog. This page explains how they get their content, how search works, and what to update when you change them. + +## Where the content comes from + +Each deployment answers from the docs that match its own code, so the docs ship inside the web app instead of being fetched from the published site. + +```text +docs/site/content/docs/*.mdx + │ make docs-catalogs (docs/site/scripts/generate-docs-bundle.ts) + ▼ +web/lib/mcp/docs/docs-bundle.snapshot.json (committed, never edited by hand) + │ + ├── web/lib/mcp/docs/bundle.ts pages + changelog, split into sections + ├── web/lib/mcp/docs/search.ts MiniSearch index, built once per server + └── web/lib/mcp/tools/docs.ts search_docs and read_doc +``` + +- The generator uses the same `getLLMText` function as `/docs/.md`, so the Markdown an agent reads matches what the site serves. +- It includes the hand-written pages, such as `cli-reference`, `mcp`, and `api` (the API overview). It leaves out the generated API endpoint pages and the MCP tool, prompt, and resource catalogs, because the client already receives the catalog when it connects. +- The `cli-reference` page includes the watcher version, so a watcher version bump changes the bundle. Run `make py-watcher-cli-catalog docs-bundle-generate` and commit both files. +- The changelog is read from `web/content/changelog/` at runtime, the same files the in-app changelog uses. Deployments get those files through `outputFileTracingIncludes` in `web/next.config.mjs`. +- `docs-lint.yml` regenerates the file and fails when the committed copy differs. After changing a docs page, run `make docs-catalogs` and commit the result. +- A change to the bundle is a change under `web/`, so the web app's Vercel project rebuilds for docs-only changes. That is intended: the deployed MCP server has to carry the new text. + +## How the agent finds the right page + +No tool returns all the docs at once. The flow is search, then read. + +1. `search_docs(query)` returns up to five matching sections, with at most two from one page. Each has a page ID, a section ID, a short excerpt, and a link to cite. +2. `read_doc(page, section?)` returns one page, or one section of it, as Markdown. It also lists the page's section IDs. It accepts a docs URL or a `/docs/...` path as well as a page ID. For an unknown page ID, the error suggests the closest page IDs, allowing for typos and other forms of a word. +3. When no section matches most of the question, `search_docs` adds a hint: the results may not answer it, so the agent should say the docs don't cover the question and offer `send_feedback` about the gap. The hint also names any words of the question that appear nowhere in the docs. +4. When nothing matches at all, `search_docs` returns every docs page with its description, and the same hint to offer `send_feedback`. + +Search ranks sections with [MiniSearch](https://lucaong.github.io/minisearch/). Headings and page titles count more than body text. Common question words and the product name are dropped, plurals are folded together, and long words tolerate a typo. Changelog entries are weighted lower so a short entry does not outrank the page that explains the feature. There are no embeddings, so there is no extra service or API key. + +A section matches when it contains any one word of the question, so most questions return results even when the docs don't cover them. A section counts as an answer when it contains at least two thirds of the question's words, rounded up, and every word of the question appears somewhere in the docs. That rule decides when the hint in step 3 appears. Some uncovered questions still pass it, for example when every word appears on an unrelated page, so the `search_docs` description also tells the agent to judge the results itself. + +Page IDs are the page's path under `docs/site/content/docs/` (`manage-tokens`, `mcp`), and changelog entries are `changelog/`. Section IDs are the `[#id]` after a heading, or the generated slug. + +## Server instructions + +`web/lib/mcp/instructions.ts` is sent to every client at connect time. Claude Code keeps only the first 2 KB, so a unit test (`tests/unit/mcp-instructions.test.ts`) fails above 2,048 bytes. Put the most important points first. + +## Usage data + +`mcp_docs_search` records how many sections answer the question by the rule above, in the same buckets as the in-app search, and `mcp_docs_read` records the page ID. The query text is never recorded. A rising share of searches in the `0` bucket shows where the docs have gaps. + +## Tests + +- `tests/unit/mcp-docs-search.test.ts` holds real questions, each with the page that should answer it. The page must be in the top three results, and the search must count at least one answer. It also holds questions the docs don't answer, which must get no answers. Add a question when someone reports a search that missed. +- `tests/unit/mcp-docs-markdown.test.ts` covers section splitting, page references, and page ID suggestions. +- `tests/mcp/mcp-protocol.test.ts` covers the tools and the resource template over the in-memory transport. +- `tests/integration/mcp.test.ts` calls both tools against the production build, which shows the docs bundle is compiled into the server. It can't show that the changelog files reach a deployment, because `next start` reads them from the source tree. + +Adding the two tools took the tool count from 37 to 39. Pinned counts in `tests/mcp/mcp-catalog.test.ts`, `tests/mcp/mcp-protocol.test.ts`, and `tests/integration/mcp.test.ts` show that change in review. diff --git a/docs/site/AGENTS.md b/docs/site/AGENTS.md index 495f586b..37ef8094 100644 --- a/docs/site/AGENTS.md +++ b/docs/site/AGENTS.md @@ -30,15 +30,26 @@ Run every command below from `docs/site/`, or use the `make docs-*` targets from ## Watcher CLI catalog - [`content/docs/cli-reference.mdx`](content/docs/cli-reference.mdx) renders `` from the committed snapshot at `src/lib/cli-catalog.snapshot.json`. -- Regenerate it with `make py-watcher-cli-catalog` (or `make docs-catalogs`, which also refreshes the other two snapshots) and commit the result. CI fails when it is out of date. A watcher version bump changes it too, because the snapshot records the version. +- Regenerate it with `make py-watcher-cli-catalog docs-bundle-generate` (or `make docs-catalogs`, which also refreshes the other two snapshots) and commit the result. CI fails when it is out of date. A watcher version bump changes it too, because the snapshot records the version. The [MCP docs bundle](#mcp-docs-bundle) embeds this page, so it changes with it. - Keep narrative caveats (interactive prompts, Windows-only service, examples) in the MDX; do not hand-maintain flag tables. - Markdown / LLM export expands `WatcherCliCatalog` via `getLLMText` (`src/lib/cli-catalog-markdown.ts`), same placeholder pattern as the MCP catalogs. The three `src/lib/*.snapshot.json` files are generated output, so Biome skips them. Never edit them by hand. +## MCP docs bundle + +The Data Hub MCP server answers questions from a copy of these pages bundled into the web app, so each deployment answers from the docs that match its own code. + +- `scripts/generate-docs-bundle.ts` writes the copy to `web/lib/mcp/docs/docs-bundle.snapshot.json`, using the same `getLLMText` output as `/docs/.md`. It leaves out the generated API endpoint pages and the MCP tool, prompt, and resource catalogs. +- Regenerate it with `make docs-catalogs` (or `npm run docs-bundle:generate` here) after changing any page or the watcher CLI snapshot, and commit the result. CI fails when it is out of date. Never edit it by hand. +- The script loads the pages through `fumadocs-mdx/node`, which needs this package to be an ES module (`"type": "module"`), and runs under `tsx`. `scripts/stub-assets-loader.mjs` stubs image imports, which Node cannot load. +- The server returns a page in one tool result, so a page must stay under 75,000 characters (the longest today is about 18,000). A test fails above that, because Claude Code caps a tool result at 25,000 tokens. Split a page that grows past it. +- Give every heading a stable `[#id]` when other pages link to it. The server uses those IDs as section names, and agents quote them back. +- Search quality is covered by a list of real questions in `web/tests/unit/mcp-docs-search.test.ts`, plus questions the docs don't answer, which must come back flagged as weak matches. When a page is renamed or split, or a new page answers one of those questions, update the lists. + ## AI-consumption surface -This site is intentionally public and indexable, and is itself built to be read by agents: `/llms.txt` (page index), `/llms-full.txt` (full corpus dump), and `/docs/.md` (any page as raw Markdown, via content negotiation on `Accept: text/markdown`). When editing docs, keep pages self-contained enough that they still make sense pulled out of the nav and read as a flat Markdown file — that's how most AI clients will actually consume them. +This site is intentionally public and indexable, and is itself built to be read by agents: `/llms.txt` (page index), `/llms-full.txt` (full corpus dump), and `/docs/.md` (any page as raw Markdown, via content negotiation on `Accept: text/markdown`). The Data Hub MCP server serves the same pages through its `search_docs` and `read_doc` tools (see [MCP docs bundle](#mcp-docs-bundle)). When editing docs, keep pages self-contained enough that they still make sense pulled out of the nav and read as a flat Markdown file — that's how most AI clients will actually consume them. ## Checks diff --git a/docs/site/content/docs/mcp/index.mdx b/docs/site/content/docs/mcp/index.mdx index 65ee5e90..dd9c7184 100644 --- a/docs/site/content/docs/mcp/index.mdx +++ b/docs/site/content/docs/mcp/index.mdx @@ -91,6 +91,17 @@ Any client that speaks Streamable HTTP and the MCP authorization spec needs one Clients that speak MCP but not OAuth can bridge through [`mcp-remote`](https://www.npmjs.com/package/mcp-remote), which runs the browser flow on their behalf. +## Questions about Data Hub [#questions-about-data-hub] + +The assistant can answer questions about Data Hub itself, such as how to install a watcher or what a run status means. It searches the documentation and the [changelog](/docs/send-feedback#changelog), reads the page that fits, and links it in its answer. Two tools do this, and both need only the `read` scope: + +- `search_docs` returns up to five matching sections, each with a short excerpt and a link +- `read_doc` returns one page, or one section of a page, as Markdown + +The answers come from the docs that ship with your deployment, so they match the version you run, including a staging deployment that is ahead of the published site. Clients that let you attach resources by hand can also attach any page as `datahub://docs/`, for example `datahub://docs/manage-tokens`. + +The assistant reads the guides and the changelog. It doesn’t read the generated [API](/docs/api) endpoint pages or the [tools](/docs/mcp/tools), [prompts](/docs/mcp/prompts), and [resources](/docs/mcp/resources) catalogs, because the client already receives the catalog when it connects. When the docs don’t cover a question, the assistant says so and offers to [send feedback](#feedback) about the gap. + ## Interactive run reports [#interactive-run-reports] Clients that support MCP Apps, an extension of the Model Context Protocol, show the same interactive run report as the web app when the assistant calls `get_run_report`. The report includes image carousels, plate maps, and the Aunty plate with its well charts. Clients without MCP Apps support get the same data as text and describe it instead. diff --git a/docs/site/package-lock.json b/docs/site/package-lock.json index 7ff48e28..866ffda5 100644 --- a/docs/site/package-lock.json +++ b/docs/site/package-lock.json @@ -44,6 +44,7 @@ "@types/react-dom": "^19.2.3", "postcss": "^8.5.15", "tailwindcss": "^4.3.1", + "tsx": "^4.23.15", "typescript": "^6.0.3", "ultracite": "^7.8.3" } @@ -5938,6 +5939,21 @@ } } }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/fumadocs-core": { "version": "16.10.5", "resolved": "https://registry.npmjs.org/fumadocs-core/-/fumadocs-core-16.10.5.tgz", @@ -9873,6 +9889,25 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tsx": { + "version": "4.23.15", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", + "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/type-fest": { "version": "5.8.0", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.8.0.tgz", diff --git a/docs/site/package.json b/docs/site/package.json index 1d869758..ad94fd15 100644 --- a/docs/site/package.json +++ b/docs/site/package.json @@ -35,10 +35,12 @@ "@types/react-dom": "^19.2.3", "postcss": "^8.5.15", "tailwindcss": "^4.3.1", + "tsx": "^4.23.15", "typescript": "^6.0.3", "ultracite": "^7.8.3" }, "name": "docs", + "type": "module", "scripts": { "build": "next build", "dev": "export VC_MICROFRONTENDS_CONFIG=../../web/microfrontends.json && next dev --port $(microfrontends port)", @@ -47,6 +49,7 @@ "typecheck": "fumadocs-mdx && next typegen && tsc --noEmit", "postinstall": "fumadocs-mdx", "check:links": "node scripts/check-links.mjs", + "docs-bundle:generate": "tsx scripts/generate-docs-bundle.ts", "lint:check": "ultracite check", "lint:fix": "ultracite fix", "precommit": "npm run lint:check && npm run typecheck" diff --git a/docs/site/scripts/generate-docs-bundle.ts b/docs/site/scripts/generate-docs-bundle.ts new file mode 100644 index 00000000..f7678ad1 --- /dev/null +++ b/docs/site/scripts/generate-docs-bundle.ts @@ -0,0 +1,52 @@ +// Writes the docs bundle the MCP server answers `search_docs` and `read_doc` +// from. Each deployment ships its own docs with the web app, so the output is +// committed under `web/`. CI regenerates it and fails when the committed copy +// is out of date. Run `make docs-catalogs` (or `npm run docs-bundle:generate` +// here) after changing a docs page or one of the other generated snapshots. +import { writeFile } from "node:fs/promises"; +import { register as registerAssetsLoader } from "node:module"; +import { flattenTree } from "fumadocs-core/page-tree"; +import { register as registerMdxLoader } from "fumadocs-mdx/node"; + +// `source.ts` imports MDX through the generated `.source/` files, which Node +// can only load with these loaders in place. +registerMdxLoader(); +registerAssetsLoader("./stub-assets-loader.mjs", import.meta.url); + +const { getLLMText, source } = await import("../src/lib/source"); + +const OUTPUT = new URL( + "../../../web/lib/mcp/docs/docs-bundle.snapshot.json", + import.meta.url +); + +// The catalog pages render from the MCP schema, which the connected client +// already receives. They would only repeat the tool list. +const EXCLUDED_SLUGS = new Set(["mcp/tools", "mcp/prompts", "mcp/resources"]); + +// Pages keep the order of the site's navigation, so the page list an agent sees +// when a search finds nothing groups related topics together. +const order = flattenTree(source.getPageTree().children).map( + (item) => item.url +); +const rank = (url: string) => { + const index = order.indexOf(url); + return index === -1 ? order.length : index; +}; + +const pages = source + .getPages() + .filter((page) => page.type !== "openapi") + .filter((page) => !EXCLUDED_SLUGS.has(page.slugs.join("/"))) + .sort((a, b) => rank(a.url) - rank(b.url) || a.url.localeCompare(b.url)); + +const entries = await Promise.all( + pages.map(async (page) => ({ + page: page.slugs.join("/"), + title: page.data.title, + description: page.data.description ?? "", + markdown: await getLLMText(page), + })) +); + +await writeFile(OUTPUT, `${JSON.stringify({ pages: entries }, null, 2)}\n`); diff --git a/docs/site/scripts/stub-assets-loader.mjs b/docs/site/scripts/stub-assets-loader.mjs new file mode 100644 index 00000000..db8f9e22 --- /dev/null +++ b/docs/site/scripts/stub-assets-loader.mjs @@ -0,0 +1,16 @@ +// Node loader hook for `generate-docs-bundle.ts`. The site's MDX options turn +// local images into static imports, which Node cannot load. The Markdown +// export only needs the original `![alt](path)` text, so image modules can be +// empty. +const IMAGE_FILE = /\.(?:png|jpe?g|gif|webp|avif|svg)$/i; + +export function load(url, context, nextLoad) { + if (url.startsWith("file:") && IMAGE_FILE.test(new URL(url).pathname)) { + return { + format: "module", + shortCircuit: true, + source: "export default { src: '', width: 0, height: 0 };", + }; + } + return nextLoad(url, context); +} diff --git a/docs/site/src/lib/mcp-catalog.snapshot.json b/docs/site/src/lib/mcp-catalog.snapshot.json index 1825c7f9..337a4c22 100644 --- a/docs/site/src/lib/mcp-catalog.snapshot.json +++ b/docs/site/src/lib/mcp-catalog.snapshot.json @@ -7,6 +7,17 @@ "endpoint": "/mcp/v1", "transport": "streamable-http" }, + "groups": { + "instruments": "Instruments", + "runs": "Runs", + "attribution": "Run attribution", + "comments": "Comments", + "files": "Files", + "watchers": "Watchers", + "discovery": "Discovery", + "docs": "Documentation", + "feedback": "Feedback" + }, "tools": [ { "name": "list_instruments", @@ -5105,6 +5116,247 @@ "additionalProperties": false } }, + { + "name": "search_docs", + "title": "Search Docs", + "description": "Search Data Hub's own documentation and changelog. Use it for how-to and explanation questions about Data Hub itself: setting up instruments and watchers, tokens, permissions, notifications, MCP and API access, and what changed recently. Returns up to five matching sections, each with a short excerpt, a page ID, and a link to cite. Call read_doc to read the full page or section. Not for lab data; use global_search or search_runs for runs, files, and instruments. When nothing matches, the result lists every docs page so you can pick one. If the results do not answer the question, say the docs do not cover it and offer to report the gap with send_feedback.", + "group": "docs", + "annotations": { + "readOnlyHint": true + }, + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "What to look up, as a question or keywords (min 2 characters)." + } + }, + "required": [ + "query" + ], + "additionalProperties": false + }, + "outputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "results": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "type": "string", + "description": "Page ID to pass to read_doc (`changelog/` for a change)." + }, + "title": { + "type": "string" + }, + "section": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Heading ID of the matching section, or null for the intro." + }, + "heading": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "excerpt": { + "type": "string" + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Link to cite. Null for a change that has no docs page." + }, + "date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "YYYY-MM-DD the change shipped. Null for a docs page." + } + }, + "required": [ + "page", + "title", + "section", + "heading", + "excerpt", + "url", + "date" + ], + "additionalProperties": false + } + }, + "pages": { + "description": "Every docs page. Present only when nothing matched.", + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "page", + "title", + "description" + ], + "additionalProperties": false + } + }, + "hint": { + "description": "Present when no section matches most of the question. Says what to do next.", + "type": "string" + } + }, + "required": [ + "results" + ], + "additionalProperties": false + } + }, + { + "name": "read_doc", + "title": "Read Doc", + "description": "Read one page of Data Hub's documentation, or one changelog entry, as Markdown. Pass a page ID from search_docs, such as `manage-tokens` or `changelog/2026-10-08-hina-large-image-previews`. Add `section` (a heading ID from the search result) to read only that part. The result lists the page's section IDs, so a long page can be read in pieces. Cite the returned link in your answer.", + "group": "docs", + "annotations": { + "readOnlyHint": true + }, + "inputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "page": { + "type": "string", + "description": "Page ID from search_docs. A docs path or URL such as /docs/manage-tokens is also accepted." + }, + "section": { + "description": "Heading ID within the page. Omit to read the whole page.", + "type": "string" + } + }, + "required": [ + "page" + ], + "additionalProperties": false + }, + "outputSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "page": { + "type": "string" + }, + "title": { + "type": "string" + }, + "section": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "heading": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "url": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "markdown": { + "type": "string" + }, + "sections": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "heading": { + "type": "string" + }, + "level": { + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991, + "description": "2 for a section, 3 for a subsection." + } + }, + "required": [ + "id", + "heading", + "level" + ], + "additionalProperties": false + }, + "description": "Heading IDs on the page, for reading one section at a time." + } + }, + "required": [ + "page", + "title", + "section", + "heading", + "url", + "markdown", + "sections" + ], + "additionalProperties": false + } + }, { "name": "send_feedback", "title": "Send Feedback", @@ -6556,6 +6808,12 @@ "mimeType": "application/json", "uriTemplate": "datahub://instruments/{instrumentId}/filter-options" }, + { + "name": "docs-page", + "description": "A page of Data Hub's own documentation as Markdown, such as datahub://docs/manage-tokens. Same text as the read_doc tool. Use search_docs to find the right page.", + "mimeType": "text/markdown", + "uriTemplate": "datahub://docs/{page}" + }, { "name": "run-report", "description": "MCP Apps view for a Data Hub instrument run report. Linked from get_run_report.", diff --git a/docs/site/src/lib/mcp-catalog.ts b/docs/site/src/lib/mcp-catalog.ts index a3ff98ae..e454baf7 100644 --- a/docs/site/src/lib/mcp-catalog.ts +++ b/docs/site/src/lib/mcp-catalog.ts @@ -30,6 +30,7 @@ export interface McpCatalogPrompt { } export interface McpCatalogDocument { + groups: Record; info: { title: string; version: string; @@ -43,18 +44,8 @@ export interface McpCatalogDocument { tools: McpCatalogTool[]; } -const GROUP_LABELS: Record = { - instruments: "Instruments", - runs: "Runs", - attribution: "Run attribution", - comments: "Comments", - files: "Files", - watchers: "Watchers", - discovery: "Discovery", -}; - export function groupLabel(group: string): string { - return GROUP_LABELS[group] ?? group; + return getMcpCatalog().groups[group] ?? group; } /** diff --git a/watcher/AGENTS.md b/watcher/AGENTS.md index 9d2ce88d..2297236b 100644 --- a/watcher/AGENTS.md +++ b/watcher/AGENTS.md @@ -12,7 +12,8 @@ If this branch changes anything under `watcher/src/` (runtime, CLI, upload, upda - **minor** (`1.0.1` → `1.1.0`) — new commands, flags, or backward-compatible features - **major** (`1.1.0` → `2.0.0`) — breaking CLI, config, or API-protocol changes 3. Run `uv lock` from the repo root so `uv.lock` records the new workspace version. -4. Do not bump again for later commits on the same branch. If the base already moved past your number, bump from the current base version instead. +4. Run `make py-watcher-cli-catalog docs-bundle-generate` from the repo root and commit both changed files. The CLI catalog snapshot records the version, and the MCP docs bundle embeds the CLI reference page. CI fails if either is out of date. +5. Do not bump again for later commits on the same branch. If the base already moved past your number, bump from the current base version instead. Do **not** bump for tests-only, docs-only, or `AGENTS.md` edits. Do **not** tag `watcher-v*` from a feature branch. Do **not** change the seeded `watcher_release_config` latest version — that is the advertised fleet pin for local seed data, not this package version. diff --git a/web/biome.jsonc b/web/biome.jsonc index cf9d7c93..92e01d3e 100644 --- a/web/biome.jsonc +++ b/web/biome.jsonc @@ -6,9 +6,10 @@ "ultracite/biome/next" ], "files": { - // Drizzle Kit writes these snapshots. Biome's JSON formatter collapses - // short arrays, and the next `drizzle-kit generate` expands them again. - "includes": ["!!drizzle/meta"] + // Drizzle Kit writes the `drizzle/meta` snapshots. Biome's JSON formatter + // collapses short arrays, and the next `drizzle-kit generate` expands them + // again. `make docs-catalogs` writes the docs bundle, so Biome skips it too. + "includes": ["!!drizzle/meta", "!lib/mcp/docs/docs-bundle.snapshot.json"] }, "linter": { "rules": { diff --git a/web/content/changelog/2026-10-09-assistant-answers-from-docs.md b/web/content/changelog/2026-10-09-assistant-answers-from-docs.md new file mode 100644 index 00000000..c7eafe37 --- /dev/null +++ b/web/content/changelog/2026-10-09-assistant-answers-from-docs.md @@ -0,0 +1,8 @@ +--- +title: AI assistants can answer questions about Data Hub from its docs +docs: https://datahub.arcadiascience.com/docs/mcp +--- + +An AI assistant connected to Data Hub can now look things up in Data Hub's documentation and in this changelog, then answer from what it finds. Ask it how to create a token, what a run status means, or what changed recently. It links the page it used, so you can read more. + +The answers come from the docs that ship with your deployment, so they match the version you run. If the docs don't cover your question, the assistant says so and offers to send feedback about the gap. It shows you the draft first. diff --git a/web/lib/analytics/track.ts b/web/lib/analytics/track.ts index 7155d0ec..2d19cab4 100644 --- a/web/lib/analytics/track.ts +++ b/web/lib/analytics/track.ts @@ -23,6 +23,19 @@ export interface AnalyticsEvents { client: string; client_version: string; }; + // Page IDs come from the docs bundle, never from what the caller typed. + mcp_docs_read: { + user_id: string; + page: string; + client: string; + }; + // Counts sections that hold most of the question's words, so `0` marks a + // question the docs likely don't cover. + mcp_docs_search: { + user_id: string; + client: string; + result_bucket: "0" | "1_to_10" | "over_10"; + }; mcp_prompt_get: { user_id: string; prompt: string; diff --git a/web/lib/feedback-app.ts b/web/lib/feedback-app.ts index 2b9c27d8..06b40e06 100644 --- a/web/lib/feedback-app.ts +++ b/web/lib/feedback-app.ts @@ -9,7 +9,7 @@ export const FEEDBACK_APP = { } as const; // The package's MCP functions name these fields differently from its server -// config, so the tool catalog and the instructions share this mapping. +// config, so the tool catalog maps them here. export const FEEDBACK_MCP_OPTIONS = { appName: FEEDBACK_APP.appName, appScope: FEEDBACK_APP.mcpAppScope, diff --git a/web/lib/mcp/analytics.ts b/web/lib/mcp/analytics.ts index cf2e5386..7ffed1ed 100644 --- a/web/lib/mcp/analytics.ts +++ b/web/lib/mcp/analytics.ts @@ -1,5 +1,10 @@ import type { AuthInfo, McpServer } from "@modelcontextprotocol/server"; -import { durationBucket, trackEvent } from "@/lib/analytics/track"; +import { + type AnalyticsEvents, + durationBucket, + searchResultBucket, + trackEvent, +} from "@/lib/analytics/track"; import { mcpClientLabel } from "@/lib/mcp/client-name"; function mcpUserId(authInfo: AuthInfo | undefined): string | undefined { @@ -11,13 +16,6 @@ interface HttpAuth { http?: { authInfo?: AuthInfo }; } -function userIdFrom(ctx: unknown): string | undefined { - if (!ctx || typeof ctx !== "object" || !("http" in ctx)) { - return; - } - return mcpUserId((ctx as HttpAuth).http?.authInfo); -} - function authInfoFrom(ctx: unknown): AuthInfo | undefined { if (!ctx || typeof ctx !== "object" || !("http" in ctx)) { return; @@ -25,24 +23,52 @@ function authInfoFrom(ctx: unknown): AuthInfo | undefined { return (ctx as HttpAuth).http?.authInfo; } -function reportTool( - tool: string, +type CallerEventName = + | "mcp_docs_read" + | "mcp_docs_search" + | "mcp_prompt_get" + | "mcp_resource_read" + | "mcp_tool_call"; + +/** + * Records an event with the caller's user ID and client label. Calls without + * a signed-in user, such as over the in-memory transport, record nothing. + */ +function trackForCaller( + name: N, ctx: unknown, - outcome: "ok" | "tool_error" | "exception", - elapsedMs: number + props: Omit ): void { - const userId = userIdFrom(ctx); + const authInfo = authInfoFrom(ctx); + const userId = mcpUserId(authInfo); if (!userId) { return; } - const authInfo = authInfoFrom(ctx); - trackEvent("mcp_tool_call", async () => ({ - user_id: userId, - tool, - client: await mcpClientLabel(authInfo), - outcome, - duration_bucket: durationBucket(elapsedMs), - })); + trackEvent( + name, + async () => + ({ + ...props, + user_id: userId, + client: await mcpClientLabel(authInfo), + }) as AnalyticsEvents[N] + ); +} + +/** + * Counts docs searches by how many sections answer the question, so a rising + * share of searches in the `0` bucket shows where the docs have gaps. The + * query text is never recorded. + */ +export function trackDocsSearch(ctx: unknown, goodMatches: number): void { + trackForCaller("mcp_docs_search", ctx, { + result_bucket: searchResultBucket(goodMatches), + }); +} + +/** Pass only a page ID that resolved against the bundle, not raw input. */ +export function trackDocsRead(ctx: unknown, page: string): void { + trackForCaller("mcp_docs_read", ctx, { page }); } function isToolError(result: unknown): boolean { @@ -154,15 +180,18 @@ export function withMcpTracking(server: McpServer): McpServer { const started = Date.now(); try { const result = await cb(...args); - reportTool( - name, - ctx, - isToolError(result) ? "tool_error" : "ok", - Date.now() - started - ); + trackForCaller("mcp_tool_call", ctx, { + tool: name, + outcome: isToolError(result) ? "tool_error" : "ok", + duration_bucket: durationBucket(Date.now() - started), + }); return result; } catch (error) { - reportTool(name, ctx, "exception", Date.now() - started); + trackForCaller("mcp_tool_call", ctx, { + tool: name, + outcome: "exception", + duration_bucket: durationBucket(Date.now() - started), + }); throw error; } }); @@ -176,29 +205,18 @@ export function withMcpTracking(server: McpServer): McpServer { // the server context last. The URI itself is never forwarded. async (...args: unknown[]) => { const ctx = args.at(-1); - const userId = userIdFrom(ctx); try { const result = await readCallback(...args); - if (userId) { - const authInfo = authInfoFrom(ctx); - trackEvent("mcp_resource_read", async () => ({ - user_id: userId, - resource: name, - client: await mcpClientLabel(authInfo), - outcome: "ok", - })); - } + trackForCaller("mcp_resource_read", ctx, { + resource: name, + outcome: "ok", + }); return result; } catch (error) { - if (userId) { - const authInfo = authInfoFrom(ctx); - trackEvent("mcp_resource_read", async () => ({ - user_id: userId, - resource: name, - client: await mcpClientLabel(authInfo), - outcome: "exception", - })); - } + trackForCaller("mcp_resource_read", ctx, { + resource: name, + outcome: "exception", + }); throw error; } } @@ -207,16 +225,8 @@ export function withMcpTracking(server: McpServer): McpServer { loose.registerPrompt = (name, config, cb) => registerPrompt(name, config, async (...args: unknown[]) => { const ctx = args.at(-1); - const userId = userIdFrom(ctx); const result = await cb(...args); - if (userId) { - const authInfo = authInfoFrom(ctx); - trackEvent("mcp_prompt_get", async () => ({ - user_id: userId, - prompt: name, - client: await mcpClientLabel(authInfo), - })); - } + trackForCaller("mcp_prompt_get", ctx, { prompt: name }); return result; }); diff --git a/web/lib/mcp/catalog/document.ts b/web/lib/mcp/catalog/document.ts index 2bc12193..a2e96159 100644 --- a/web/lib/mcp/catalog/document.ts +++ b/web/lib/mcp/catalog/document.ts @@ -3,7 +3,10 @@ import { zodTypeToJsonSchema, } from "@/lib/mcp/catalog/json-schema"; import { MCP_TOOL_DEFS } from "@/lib/mcp/catalog/tools"; -import type { McpCatalogDocument } from "@/lib/mcp/catalog/types"; +import { + MCP_TOOL_GROUPS, + type McpCatalogDocument, +} from "@/lib/mcp/catalog/types"; import { MCP_PROMPT_DEFS } from "@/lib/mcp/prompts.defs"; import { MCP_RESOURCE_DEFS } from "@/lib/mcp/resources.defs"; @@ -18,6 +21,7 @@ export function buildMcpCatalogDocument(): McpCatalogDocument { endpoint: "/mcp/v1", transport: "streamable-http", }, + groups: MCP_TOOL_GROUPS, tools: MCP_TOOL_DEFS.map((tool) => ({ name: tool.name, title: tool.title, diff --git a/web/lib/mcp/catalog/tools.ts b/web/lib/mcp/catalog/tools.ts index d9c29402..75444164 100644 --- a/web/lib/mcp/catalog/tools.ts +++ b/web/lib/mcp/catalog/tools.ts @@ -1,5 +1,6 @@ import type { McpToolDef } from "@/lib/mcp/catalog/types"; import { DISCOVERY_TOOL_DEFS } from "@/lib/mcp/tools/discovery.defs"; +import { DOCS_TOOL_DEFS } from "@/lib/mcp/tools/docs.defs"; import { FEEDBACK_TOOL_DEFS } from "@/lib/mcp/tools/feedback.defs"; import { FILE_TOOL_DEFS } from "@/lib/mcp/tools/files.defs"; import { INSTRUMENT_TOOL_DEFS } from "@/lib/mcp/tools/instruments.defs"; @@ -15,5 +16,6 @@ export const MCP_TOOL_DEFS: readonly McpToolDef[] = [ ...FILE_TOOL_DEFS, ...WATCHER_TOOL_DEFS, ...DISCOVERY_TOOL_DEFS, + ...DOCS_TOOL_DEFS, ...FEEDBACK_TOOL_DEFS, ]; diff --git a/web/lib/mcp/catalog/types.ts b/web/lib/mcp/catalog/types.ts index e27f74e6..f2720cb0 100644 --- a/web/lib/mcp/catalog/types.ts +++ b/web/lib/mcp/catalog/types.ts @@ -9,6 +9,7 @@ export const MCP_TOOL_GROUPS = { files: "Files", watchers: "Watchers", discovery: "Discovery", + docs: "Documentation", feedback: "Feedback", } as const; @@ -63,6 +64,8 @@ export type McpResourceDef = { /** Shape served by `GET /mcp/v1/schema.json`. */ export interface McpCatalogDocument { + /** Display label for each tool `group`. */ + groups: typeof MCP_TOOL_GROUPS; info: { title: string; version: string; diff --git a/web/lib/mcp/docs/bundle.ts b/web/lib/mcp/docs/bundle.ts new file mode 100644 index 00000000..966d9301 --- /dev/null +++ b/web/lib/mcp/docs/bundle.ts @@ -0,0 +1,214 @@ +import { listChangelogEntries } from "@/lib/changelog/entries"; +import snapshot from "./docs-bundle.snapshot.json"; +import { type MarkdownSection, splitSections, toPlainText } from "./markdown"; + +const CHANGELOG_PREFIX = "changelog/"; +const MAX_SUGGESTIONS = 3; + +export interface DocSection extends MarkdownSection { + plainText: string; +} + +interface DocEntryBase { + /** Plain text before the first `##` heading, or "" when there is none. */ + introText: string; + markdown: string; + /** `manage-tokens` for a docs page, `changelog/` for a change. */ + page: string; + sections: DocSection[]; + title: string; +} + +export interface DocsPageEntry extends DocEntryBase { + description: string; + kind: "docs"; +} + +export interface ChangelogDocEntry extends DocEntryBase { + /** Day the change shipped, as YYYY-MM-DD. */ + date: string; + /** The entry's own `docs` link, if it has one. */ + docsUrl: string | null; + kind: "changelog"; +} + +export type DocEntry = DocsPageEntry | ChangelogDocEntry; + +export interface DocPageSummary { + description: string; + page: string; + title: string; +} + +function splitPage( + markdown: string +): Pick { + const { intro, sections } = splitSections(markdown); + return { + introText: toPlainText(intro), + sections: sections.map((section) => ({ + ...section, + plainText: toPlainText(section.text), + })), + }; +} + +function buildBundle(): DocEntry[] { + const pages = snapshot.pages.map( + (page): DocsPageEntry => ({ + kind: "docs", + page: page.page, + title: page.title, + description: page.description, + markdown: page.markdown, + ...splitPage(page.markdown), + }) + ); + + const changes = listChangelogEntries().map((entry): ChangelogDocEntry => { + const markdown = `# ${entry.title} (${entry.date})\n\n${entry.body}${ + entry.docs ? `\n\nRelated docs: ${entry.docs}` : "" + }`; + return { + kind: "changelog", + page: `${CHANGELOG_PREFIX}${entry.id}`, + title: entry.title, + date: entry.date, + docsUrl: entry.docs ?? null, + markdown, + ...splitPage(markdown), + }; + }); + + return [...pages, ...changes]; +} + +// Docs are fixed per deployment, and so is the changelog; dev skips the cache +// so a new changelog file shows up without a restart. +let cached: DocEntry[] | null = null; + +export function getDocsBundle(): DocEntry[] { + if (process.env.NODE_ENV === "production" && cached) { + return cached; + } + const bundle = buildBundle(); + if (process.env.NODE_ENV === "production") { + cached = bundle; + } + return bundle; +} + +export function listDocPages(): DocPageSummary[] { + return getDocsBundle() + .filter((entry): entry is DocsPageEntry => entry.kind === "docs") + .map(({ page, title, description }) => ({ page, title, description })); +} + +export function findDoc(page: string): DocEntry | undefined { + return getDocsBundle().find((entry) => entry.page === page); +} + +/** Error text that lets an agent correct a wrong page ID in one more call. */ +export function unknownPageMessage(page: string): string { + const pages = listDocPages().map((doc) => doc.page); + const close = closestPages(page.toLowerCase(), pages); + return [ + `Page '${page}' not found.`, + close.length > 0 ? `Closest: ${close.join(", ")}.` : null, + `Docs pages: ${pages.join(", ")}.`, + "Changelog entries are named changelog/; find them with search_docs.", + ] + .filter(Boolean) + .join(" "); +} + +// Catches a typo (`manage-tokns`), a missing word (`install-watcher`), and +// another form of a word (`troubleshooting`), closest first. +function closestPages(typed: string, pages: readonly string[]): string[] { + if (typed.length < 2) { + return []; + } + return pages + .map((page) => ({ page, distance: pageDistance(typed, page) })) + .filter( + (candidate): candidate is { page: string; distance: number } => + candidate.distance !== null + ) + .sort((a, b) => a.distance - b.distance) + .slice(0, MAX_SUGGESTIONS) + .map(({ page }) => page); +} + +function pageDistance(typed: string, page: string): number | null { + if (page.includes(typed) || typed.includes(page)) { + return 0; + } + const distance = editDistance(typed, page); + const allowed = Math.max(2, Math.floor(page.length / 4)); + return distance <= allowed || sharesWordStem(typed, page) ? distance : null; +} + +function sharesWordStem(a: string, b: string): boolean { + const words = (text: string) => + text.split(/[-/]/).filter((word) => word.length >= 4); + return words(a).some((left) => + words(b).some((right) => left.startsWith(right) || right.startsWith(left)) + ); +} + +/** Levenshtein distance: the fewest single-letter edits from `a` to `b`. */ +function editDistance(a: string, b: string): number { + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i++) { + const current = [i]; + for (let j = 1; j <= b.length; j++) { + const substitution = a[i - 1] === b[j - 1] ? 0 : 1; + current[j] = Math.min( + previous[j] + 1, + current[j - 1] + 1, + previous[j - 1] + substitution + ); + } + previous = current; + } + return previous[b.length]; +} + +export interface DocRef { + page: string; + section?: string; +} + +const DOCS_PATH = /^\/?docs\/(.+)$/; + +/** + * Accepts what an agent is likely to pass: a page id, a path such as + * `/docs/manage-tokens`, a full docs URL, or any of those with a `#section` + * or `.md` suffix. An explicit `section` argument wins over a URL fragment. + */ +export function parseDocRef(page: string, section?: string): DocRef { + let path = page.trim(); + let fragment: string | undefined; + + if (/^https?:\/\//i.test(path)) { + try { + const url = new URL(path); + path = url.pathname; + fragment = url.hash.slice(1) || undefined; + } catch { + // Leave the text as is; the lookup reports it as an unknown page. + } + } + + const hashAt = path.indexOf("#"); + if (hashAt !== -1) { + fragment ||= path.slice(hashAt + 1) || undefined; + path = path.slice(0, hashAt); + } + + path = path.replace(DOCS_PATH, "$1").replace(/^\/+|\/+$/g, ""); + path = path.replace(/\.md$/i, ""); + + const wanted = section?.trim() || fragment; + return wanted ? { page: path, section: wanted } : { page: path }; +} diff --git a/web/lib/mcp/docs/docs-bundle.snapshot.json b/web/lib/mcp/docs/docs-bundle.snapshot.json new file mode 100644 index 00000000..d0ab22d0 --- /dev/null +++ b/web/lib/mcp/docs/docs-bundle.snapshot.json @@ -0,0 +1,160 @@ +{ + "pages": [ + { + "page": "overview", + "title": "Overview", + "description": "What Data Hub does, what it doesn't do, where your data shows up, and which page to read next.", + "markdown": "# Overview (/docs/overview)\n\n\n\n**For:** Everyone\n\nInstrument data tends to stay where it was made. Files sit on the PC wired to the plate reader, named by whoever was at the bench, backed up if someone remembers. Finding a result from three months ago means knowing which machine made it and who was running it.\n\nData Hub moves that data off the instrument PC automatically. A small program called a watcher runs on each instrument PC, monitors its output folder, uploads new files as they appear, and groups them into runs. From then on the data is searchable in a browser, tied to the person who ran it, and reachable from a script or an AI agent without anyone copying files to a shared drive.\n\n## What Data Hub isn’t [#what-data-hub-isnt]\n\nWorth setting expectations early, because the name invites a few wrong guesses:\n\n* It doesn’t change your raw files. Uploads are byte-for-byte, and processing only ever adds new files alongside them\n* It isn’t an electronic lab notebook. There’s a comment thread per run for notes, and that’s the extent of it\n* It isn’t analysis software. It parses vendor formats into something viewable and downloadable; the analysis is still yours\n* It doesn’t run on the instrument. Processing happens server-side after upload, so nothing competes with acquisition software for resources\n\n## How a file becomes a run [#how-a-file-becomes-a-run]\n\n1. A [watcher](/docs/concepts#watcher) on the instrument PC notices a new file in the folder it’s watching, waits for the file to stop changing, and groups it with the other files from the same acquisition into a [run](/docs/concepts#run).\n2. The watcher uploads those files to cloud storage. For [supported instrument types](/docs/instrument-preprocessing), server-side code then reads the vendor’s format, pulls out settings like wavelength or imaging mode, and builds viewable versions such as plate maps or contrast-adjusted images.\n3. The run appears in the web app, the REST API, and the Model Context Protocol (MCP) endpoint at the same time. They’re three views of one record.\n\nThe full picture, including where each piece runs, is in [Architecture](/docs/architecture).\n\n## Where your data shows up [#where-your-data-shows-up]\n\nFive places in the web app, in the order most people use them:\n\n* **Home**: every instrument’s recent runs together, with counts for today and this week, and today’s comments\n* **Instruments**: one instrument’s runs, with columns matching what that instrument records\n* **My runs**: runs you’ve claimed, so you can find your own work without filtering\n* **Comments**: every run comment by day, filtered to your runs or to one instrument if you like\n* **Watchers**: which instrument PCs are reporting in, for whoever maintains them\n\nTo see what has changed in Data Hub recently, open **Changelog** from the account menu.\n\nIf a term on those pages is unfamiliar, [Concepts](/docs/concepts) defines the vocabulary in one page. Run, watcher, run status, and instrument type are the four worth reading first.\n\n## Get at the same data from code [#get-at-the-same-data-from-code]\n\nThe web app has no private back door. Everything it shows comes from a REST API that you can call yourself, and an MCP endpoint that AI clients can call on your behalf. Both read the same records, so a script sees exactly what the dashboard sees.\n\n* **REST**: served at `/api/v1/` on your own deployment host. Each deployment publishes its own OpenAPI 3.1 document at `/api/v1/openapi.json`, so you can generate a client rather than hand-write one. See the [API overview](/docs/api)\n* **MCP**: served at `/mcp/v1` over Streamable HTTP, and added to Claude Code, Claude Desktop, or Cursor by pasting the URL. Agents can then search runs, read metadata, and fetch files during a conversation. Clients that support MCP Apps also show the same interactive run reports as the web app. See the [MCP overview](/docs/mcp)\n\nThe two differ in how you authenticate. REST clients carry a [personal access token](/docs/concepts#personal-access-token) that an admin creates in the web app, scoped and revocable. A token acts under its own name, so anything a script does is labeled with the token rather than with a person. MCP clients send you through a browser sign-in and a consent screen instead, then act as you with your own permissions, so nobody has to paste a credential into an AI client.\n\nTypical uses look like pulling every run from an instrument for a batch reanalysis, writing the latest run’s archive to disk from a script, or asking an agent which plates from last week failed processing.\n\n## What’s supported today [#whats-supported-today]\n\nNine instrument types have a processor, covering plate readers, gel docs, qPCR, TapeStation, FPLC, flatbed plate scanners, ND2 microscopy, DishCam time-lapse photos, and the Unchained Labs Aunty for protein melting temperature and particle size. [Instrument data preprocessing](/docs/instrument-preprocessing) lists what each one extracts.\n\nAn instrument without a matching processor still works. Its files upload, and you can search, claim, comment on, and download them. What you don’t get is extracted metadata, badges, or rendered previews. Adding a new type is a development task, not a settings change.\n\n## Self-hosted by design [#self-hosted-by-design]\n\nData Hub isn’t a service you sign up for. Your team runs it, which is the point: raw instrument data and everything derived from it stay on infrastructure you control, and the source is open under the MIT license.\n\nRunning it means four pieces, deployed once by an engineer:\n\n* The web app, which also serves the API and the MCP endpoint, on Vercel\n* A PostgreSQL database for instruments, runs, files, and comments\n* AWS S3 for the files themselves, split into raw and processed buckets\n* An AWS Lambda that does the per-instrument processing and builds download archives\n\nA Google OAuth client, created during that deploy, is the fifth piece and the one worth knowing about even if you’ll never touch the stack. It decides who can sign in at all: only your Google Workspace organization, or any Google account. Nothing inside Data Hub overrides that choice, so it’s a deployment decision rather than a setting an admin can change later.\n\nUntil the stack exists, nobody can sign in and no watcher can upload. The deployment runbooks live in the repository rather than on this site; [Deploy Data Hub](/docs/self-hosting) points to them.\n\n## Getting access [#getting-access]\n\nSign in with your Google account, assuming it’s one the deployment allows. Everyone who signs in can browse instruments, runs, and files, comment on runs, and [send feedback](/docs/send-feedback) about Data Hub. Admins can additionally confirm instruments, issue tokens, promote other users, and connect Linear and Slack, and only an existing admin can grant you that. See [Security and permissions](/docs/security#roles) for what each role can do.\n\n## Start here [#start-here]\n\n\n \n\n \n\n \n\n \n\n\n## Reading these docs with AI [#reading-these-docs-with-ai]\n\nEvery page has a **Copy Markdown** button and a **View Options** menu. Use them to:\n\n* Copy a page as Markdown into an assistant\n* Open raw Markdown (`/docs/.md`) or the dumps at [`/docs/llms.txt`](/docs/llms.txt) and [`/docs/llms-full.txt`](/docs/llms-full.txt)\n" + }, + { + "page": "self-hosting", + "title": "Deploy Data Hub", + "description": "What deploying Data Hub involves, what you need before you start, and where the step-by-step runbooks live.", + "markdown": "# Deploy Data Hub (/docs/self-hosting)\n\n\n\n**For:** Engineers\n\nData Hub is self-hosted, so nobody can sign in or install a watcher until your team has stood up the backend. This page is orientation only: what gets deployed, what access you need, and in what order. It doesn’t walk through the commands.\n\n\n For the actual deploy instructions, open [First-time deployment](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md) in the data-hub repository. That guide has every command, env var, and checklist. Read this page first if you want the map before you start.\n\n\n## What you’ll deploy [#what-youll-deploy]\n\nThree pieces, each on infrastructure you choose:\n\n| Piece | What it does | Where it runs |\n| ---------------------- | ----------------------------------------------------------------------------------------- | ---------------------------------------- |\n| PostgreSQL database | Stores instruments, runs, files, watchers, and tokens | Any PostgreSQL host, managed or self-run |\n| Web app | Serves the dashboard, the REST API, and the MCP endpoint | Vercel |\n| Storage and processing | S3 buckets for raw, processed, and archived files, plus the Lambda that processes uploads | AWS, deployed with AWS SAM |\n\nYou’ll also create a Google OAuth client, which is what actually controls who can reach your deployment. Data Hub has no invite flow and no email allowlist of its own, so that client’s setting is the whole access policy. Decide it deliberately and read [Who can sign in](/docs/security#who-can-sign-in) first.\n\nGive `staging` and `production` separate databases and separate AWS stacks. They’re independent deployments holding independent data.\n\n## Access you need before you start [#access-you-need-before-you-start]\n\nThe deploy stalls without all four, so confirm them up front:\n\n* **Admin-level AWS credentials.** The first deploy creates the stack, which the automated deploy role can’t do. It can only update a stack that already exists\n* **Vercel project access**, including permission to set environment variables for the environment\n* **Google Cloud Console access** to create the OAuth client\n* **GitHub repository settings access**, to store the secrets that let continuous integration handle every deploy after this one\n\nYou’ll also need the AWS CLI and the AWS SAM CLI installed locally.\n\n## How the deploy is sequenced [#how-the-deploy-is-sequenced]\n\nThe web app and the AWS stack each need something the other produces, so the order matters. This is the sequence only; [First-time deployment](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md) has the commands for each step:\n\n\n \n **Create the database** and keep its connection string. Make sure the host accepts connections from Vercel and from your continuous integration runners.\n \n\n \n **Deploy the web app**, set its environment variables, apply the database migrations, then sign in and create the access token the Lambda will use.\n \n\n \n **Bootstrap AWS resources**, once per AWS account rather than once per environment. This creates the shared resources both environments draw on.\n \n\n \n **Deploy the AWS stack** for this environment, using the web app’s URL and that token.\n \n\n \n **Wire the AWS outputs back into the web app**, then redeploy so it picks them up.\n \n\n\nExpect the web app to deploy cleanly at step 2 and still not work. Anything touching files, uploads, reprocessing, and run archives stays broken until step 5 connects it to AWS. A green deployment is not the finish line.\n\n## Things that catch people out [#things-that-catch-people-out]\n\nThese failures account for most lost time on a first deploy:\n\n* **Set `ADMIN_EMAILS` before you sign in for the first time.** It’s what promotes the first admin. Sign in before setting it and you get an account with no permission to create tokens or confirm instruments\n* **Point `BETTER_AUTH_URL` at this deployment’s own public origin.** Data Hub builds the Google callback URL from it, so a value copied from another environment sends people to that other host and sign-in fails. AI clients read the same value when they sign in over MCP, so a wrong one breaks them too\n* **Google sign-in on Vercel preview deployments needs `OAUTH_PROXY_URL` and `OAUTH_PROXY_SECRET`.** Google matches each redirect URI exactly and every preview gets a new URL, so previews route the callback through one stable deployment whose URI is registered. Production registers its own and leaves both variables unset\n* **Create the Lambda’s token from the Lambda preset.** The Lambda checks an instrument’s type before choosing a processor, so without `instruments:read` every upload fails on a permission error rather than an obvious misconfiguration. DishCam processing also reads the run’s file list, so it fails without `runs:read`. The **Lambda** preset includes both\n* **Set `ENABLE_S3_FILES` to `true` or `false` for the AWS stack.** An empty value fails the deploy. Use `false` unless you want the Lambda to read very large files from a mounted copy of the raw bucket, which needs a VPC and costs extra; the runbook covers turning it on later\n* **Bootstrapping fails if the AWS account already has a GitHub identity provider** from an unrelated project. Reuse the existing one instead of creating a second\n* **Set `INTEGRATION_SECRETS_KEY` before an admin saves Slack or Linear secrets.** Data Hub encrypts those secrets with it, and saving fails without it. Generate one with `openssl rand -hex 32`\n* **The Slack channel webhook and the Linear connection aren’t environment variables.** A workspace admin sets them under **Settings → Integrations** after the deploy; see [Set up integrations](/docs/integrations). The Slack app credentials for direct messages can be either environment variables or saved there. Feedback stays turned off until an admin connects Linear\n\n## What finished looks like [#what-finished-looks-like]\n\nOperators can install watchers and instruments start uploading. Hand them [Set up an instrument](/docs/set-up-an-instrument), and create their first tokens through [Issue and revoke tokens](/docs/manage-tokens). To turn on feedback reports and Slack messages, an admin then follows [Set up integrations](/docs/integrations).\n\nFrom then on, deploys are automatic. Merges deploy the web app, apply migrations, and ship the Lambda, so this manual sequence runs once per environment.\n\n## The runbooks [#the-runbooks]\n\nThe deploy commands live next to the code in the data-hub repository. Start with [First-time deployment](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md):\n\n* [First-time deployment](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md): the full sequence above, with every command\n* [Continuous integration and deployment](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/ci-and-deployment.md): how later deploys, migrations, and watcher releases happen\n* [Local development](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/local-development.md): run the app on a laptop, including a setup that needs no AWS or Google credentials\n* [Architecture for contributors](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/architecture.md): internals\n\nFor a product-level map of the components and how data moves between them, rather than deploy steps, see [Architecture](/docs/architecture).\n" + }, + { + "page": "set-up-an-instrument", + "title": "Set up an instrument", + "description": "Go from a fresh instrument PC to an active instrument uploading runs into Data Hub.", + "markdown": "# Set up an instrument (/docs/set-up-an-instrument)\n\n\n\n**For:** Lab operators and admins\n\nThis walkthrough takes you from nothing to an uploading instrument: create an access token, install the watcher, register the instrument, get an admin to confirm it, then start watching. CLI flags, the full config schema, and Windows service setup each have their own page; this guide links them once.\n\n\n Data Hub is self-hosted. The web app, PostgreSQL database, and AWS stack (S3 plus the processing Lambda) must already be running. If they are not, an engineer needs the [Deploy Data Hub](/docs/self-hosting) guide first.\n\n\n## Before you start [#before-you-start]\n\nYou need:\n\n* **The watch directory**: the folder your instrument writes output into.\n* **File patterns**: globs to upload, for example `*.csv`, `*.xlsx`, `*.tiff`.\n* **[uv](https://docs.astral.sh/uv/getting-started/installation/)** installed on the instrument PC (it manages Python for you).\n\nTo make setup and maintenance easier, we recommend setting up remote access to the instrument PC via [Windows Remote Desktop](https://learn.microsoft.com/en-us/windows-server/remote/remote-desktop-services/remotepc/remote-desktop-allow-access).\n\n## 1. Create an access token [#1-create-an-access-token]\n\nThe watcher authenticates to Data Hub with a personal access token. Only workspace admins can create tokens, so ask an admin if you are not one.\n\n\n \n In the Data Hub web app, go to **Settings → Access Tokens** and click **Create token**.\n \n\n \n Give the token a **Name** that identifies the instrument PC, for example `Plate Reader PC`.\n \n\n \n Set **Expiration** to “No expiry”. (The token can always be revoked if necessary.)\n \n\n \n Under **Scopes**, click the **Watcher** preset. It grants only what a watcher needs: registering, reporting runs, and uploading files.\n \n\n \n Click **Create token**, then copy the token. It starts with `dhub_` and is shown only once.\n \n\n\nKeep the token handy: you paste it into the setup wizard in step 3. For rotation, revocation, and the full scope list, see [Issue and revoke tokens](/docs/manage-tokens).\n\n## 2. Install the watcher [#2-install-the-watcher]\n\nOn the instrument PC, install the watcher with `uv`. The `windows-service` extra adds `pywin32` so you can follow [Run as a Windows service](/docs/windows-service), which is the default on a lab PC:\n\n```sh\nuv tool install \"data-hub-watcher[windows-service]\"\n```\n\nOn macOS or Linux, omit the extra:\n\n```sh\nuv tool install data-hub-watcher\n```\n\nFull install, verify, and update steps live in [Install and update the watcher](/docs/install-the-watcher).\n\n## 3. Run the setup wizard [#3-run-the-setup-wizard]\n\nStart the wizard with `data-hub-watcher init`. It prompts you for one thing at a time:\n\n\n \n Choose the environment. For a live lab, this is production.\n \n\n \n Enter your Data Hub API URL e.g. `https://datahub.your-domain.com/api/v1`.\n \n\n \n Paste the token from step 1 when it asks for the API key. The wizard checks the key, then lists the instruments already registered.\n \n\n \n Pick your instrument from the list, or choose **Register a new instrument** if it is not there.\n \n\n \n For a new instrument, enter an ID and a display name. The ID is a short, lowercase, hyphenated name like `bio-rad-cfx96`. **It cannot be changed later.**\n \n\n \n Enter the full path to the folder your instrument writes into.\n \n\n \n List which files to upload, such as `*.pcrd,*.csv`. The watcher ignores files that do not match.\n \n\n \n Choose how files get grouped into runs. Pick the preset that matches how your instrument names its output, or write your own. The wizard previews a few matches so you can check the run IDs. For example, the default preset uses the filename prefix before the first underscore, so `RUN001_quant.csv` and `RUN001_melt.csv` both group into run `RUN001`.\n \n\n \n Set the stability period. This is how much time (in seconds) a file must stay unchanged before the watcher counts it as finished. The default is 5 seconds; raise it for instruments that write large files slowly.\n \n\n \n Choose the upload mode. “Auto” uploads each file as soon as it is stable; “Manual” waits for your approval first.\n \n\n\nFor run detection options, see [Configure run detection and uploads](/docs/configure-run-detection); [Watcher configuration](/docs/configuration-reference#run-detection) lists every preset.\n\nThe wizard then registers the watcher, saves the config to `~/.data-hub/config.yaml` (`C:\\Users\\\\data-hub\\config.yaml` on Windows), and syncs it to Data Hub. When it reports that setup is complete, start watching.\n\n## 4. Confirm the instrument’s registration [#4-confirm-the-instruments-registration]\n\nA pending instrument cannot upload to Data Hub until an admin confirms it from the web app.\n\n\n \n Open the Data Hub web app and navigate to **Instruments > View all** from the app sidebar.\n \n\n \n Select the “Pending” tab and find your instrument. It should show a yellow `pending` badge.\n \n\n \n Click **Confirm**. The instrument should move to the “Active” tab and appear as online shortly.\n \n\n\nOpen the instrument’s page and check the watched folder and file patterns shown under its name. They should match what you entered in the wizard; files outside that folder, or with names that don’t match a pattern, won’t reach Data Hub.\n\nSome instruments also get automated preprocessing (metadata extraction, image conversion, and related outputs). To choose a processor, an admin opens the three-dot menu, selects **Edit details**, and picks a supported instrument type. What each type produces is in [Instrument data preprocessing](/docs/instrument-preprocessing).\n\n\n Adding a new processor requires code changes; see [Adding a new instrument](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/lambda.md#adding-a-new-instrument) in the developer docs.\n\n\nSee [Manage instruments](/docs/manage-instruments) for more info.\n\n## 5. Start the watcher service [#5-start-the-watcher-service]\n\nFirst, preview what the watcher will do without starting it:\n\n```sh\ndata-hub-watcher watch --dry-run\n```\n\nThis validates config, checks that the API is reachable and the instrument is active, and previews which files would be picked up.\n\nWhen it looks right, install and start the watcher as a Windows service. Run these from an **Administrator** terminal:\n\n```sh\ndata-hub-watcher service install\ndata-hub-watcher service start\n```\n\nThe service starts on boot, keeps running after you log out, and auto-updates in the background. Lifecycle and log details are in [Run as a Windows service](/docs/windows-service).\n\nOn macOS or Linux, run the watcher in the foreground instead:\n\n```sh\ndata-hub-watcher watch\n```\n\nThe watcher detects new files, groups them into runs, uploads them (in auto mode), and sends a heartbeat every 60 seconds. Runs appear in the dashboard after upload.\n\n## What you get once it is running [#what-you-get-once-it-is-running]\n\n* Raw files upload to cloud storage\n* Instrument runs and files appear in the web dashboard\n* Watcher service automatically restarts on PC restart\n* Watcher health (heartbeats, events) appears under **Watchers**\n* Subscribed users can get in-app and Slack notifications for new runs\n" + }, + { + "page": "browse-runs", + "title": "Browse and analyze runs", + "description": "Use the web app to find runs, inspect processed results, claim attribution, download files, comment, and manage run lifecycle.", + "markdown": "# Browse and analyze runs (/docs/browse-runs)\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n**For:** Everyone\n\nAfter instruments are uploading, the web app is where you find runs and act on them: inspect results, claim work, download files, and leave lab notes. The sections below follow the order you’ll use them in.\n\n## Orient yourself in the app [#find-a-run]\n\nThe sidebar and the home page are the two starting points for daily work.\n\n**Home** lists recent runs across instruments, most recent first, with counts for today and this week at the top. Filter by date range, who ran it, and instrument-specific metadata when it’s available, then page through results. Below the runs, Home shows up to four of today’s comments; use the date picker to see comments from a different range.\n\n\"Home\n\n**Instruments** in the sidebar lists five instruments, and **View all** opens the full list. Each instrument’s page shows the folder its watcher reads from and the file patterns it uploads, then a runs table with column filters that match the instrument type (for example wavelengths on a plate reader).\n\n\"Plate\n\n**My runs** lists runs you’ve claimed, so you can return to your own work without re-filtering the full dashboard.\n\n\"My\n\n**Comments** lists every run comment by day. Open it from **View all comments** on Home. Each person’s profile page also shows the comments they wrote and the comments on runs they ran.\n\n### Narrow a long run list [#narrow-a-long-run-list]\n\nHome and the instrument pages share the same four controls:\n\n* **Search runs**: filters the list by run name as you type\n* **Date range**: presets from Today through Last 4 weeks, plus a custom range\n* **Add filter**: filter by run status, or turn on **Show deleted runs**\n* **Column headers**: sort by **Run Started**, **Last Updated**, size, who ran it, or any instrument-specific column\n\n**Run Started** is when the instrument acquired the run. **Last Updated** changes when the run’s files change, or when someone edits, claims, or comments on it. The default order is newest run first. The sort order is saved in the page link, so a link you share opens with the same order.\n\n### What each run status means [#run-status]\n\nA run’s status tells you how far its files have got from the instrument to the dashboard:\n\n| Status | Meaning |\n| -------------- | ------------------------------------------------------------------------------------------------------------ |\n| Failed | At least one file failed processing. The file’s row on the run page gives the reason. |\n| Stalled | At least one file has been processing for more than 20 minutes without finishing. Reprocess it to try again. |\n| Pending upload | The watcher found the files but hasn’t sent them yet. |\n| Uploaded | The files are in storage. Processing hasn’t started, or this instrument type has no processor. |\n| Processing | The server is reading the files, extracting metadata, and building reports. |\n| Completed | Processing finished. |\n| Empty | The run has no files. |\n\nA run with files in several states shows the first status in this table that applies. For example, a run with one failed file and nine completed files shows **Failed**.\n\n## Search the workspace [#search]\n\nPress **⌘ K** (macOS) or **Ctrl K** (Windows/Linux) to open global search from any page. Results are grouped into runs, files, instruments, users, and comments. That means you can also look up a colleague’s work or the text of a lab note. A file hit opens the parent run with the files table already filtered to that file.\n\n\"Global\n\n## Open a run and inspect results [#open-run-detail]\n\nEach run has a details page at `/instruments/your_instrument_id/runs/your_run_id`. The arrows beside the run name step you to the next or previous run without going back to the list.\n\nThe top of the page lists who ran it, the number of raw files, and their total size. What you see below that depends on the instrument type and whether a processor ran:\n\n* Metadata tags for extracted fields, such as experiment type, wavelength, or temperature\n* Processed reports such as tables, images, PDFs, and videos, including interactive plate maps for plate readers and a video player for DishCam runs\n* A files table of everything in the run, labeled Raw or Processed, with each file’s size and status\n\nClick a metadata tag to open the instrument’s runs table filtered to that value. For example, clicking a wavelength tag lists every run on that instrument recorded at the same wavelength.\n\nReports with many images, PDFs, or spectra show one item at a time. Search by filename to jump anywhere in the run, or use the arrows to step through it. The counter beside the arrows shows your position out of the total. Plate maps with more than 12 columns, such as 384-well plates, scroll sideways inside the plate.\n\nPer-type inputs and outputs are in [Instrument data preprocessing](/docs/instrument-preprocessing).\n\n\"Gel\n\n\"Plate\n\n\"Plate\n\n### Filter the files table [#filter-files]\n\nThe files table has separate **Type** and **Status** filters. Use them to show, for example, only failed files or only processed CSVs. **Download all** and the run archive include only the files the filters show.\n\n### Live file updates [#live-file-updates]\n\nWhile files are uploading or processing, the run page checks for updates every 5 seconds for up to 5 minutes. After that it stops and shows **Stopped checking for file updates**. Choose **Refresh** to check for another 5 minutes. Uploading or reprocessing files from the page also starts a new 5 minutes.\n\n## Claim who ran it [#claim-who-ran-it]\n\nUse **Claim** on the run details page to attribute yourself. You can claim or unclaim your own user only, so nobody else can attribute a run to you. Claims show on the run and appear in “ran by” filters on Home and elsewhere. API tokens can’t claim runs; a signed-in person has to.\n\n## Download files [#download-files]\n\nFrom the files table you can:\n\n* Download a single file, which opens a temporary download link\n* Select several files and choose **Download** in the bar that appears at the bottom of the page\n* Choose **Download all** to build a zip of active files in the run, with any active file filters applied\n\nLarge zips build in the background. When one isn’t ready yet, Data Hub tells you how long to wait before trying again.\n\n## Act on several runs at once [#bulk-actions]\n\nSelect the checkbox on one or more run rows to open an action bar at the bottom of the list. Run rows have no buttons of their own; every run action goes through this bar.\n\n* **I ran these**: claim every selected run in one step\n* **Remove my attribution**: unclaim them\n* **Upload**: ask the watcher to upload files that are still on the instrument PC\n* **Download**: download a zip of each selected run\n* **Reprocess**: run server-side processing again, including on runs whose files have stalled\n* **Delete**: mark the selected runs deleted\n\n**Upload**, **Download**, **Reprocess**, and **Delete** appear only when every selected run allows them. For example, **Reprocess** is hidden if one of the selected runs belongs to an instrument type with no processor. The checkbox in the header row selects every run on the page, and **Clear** drops the selection.\n\nThe files table on a run page works the same way: select files to get **Upload**, **Dismiss**, **Reprocess**, and **Download** for just those files.\n\n\"Run\n\n## Comment on a run [#comment]\n\nThe comments thread on run details is for lab notes and follow-ups. Comment activity can notify people who claimed the run or already commented, and clicking a comment notification scrolls to that comment. Set preferences under **Settings → Notifications**; see [Configure notifications](/docs/configure-notifications).\n\nOnly the author of a comment can edit it. Authors can delete their own comments, and workspace admins can delete any comment. Comments posted by a script through an API token show the token’s name with a key icon instead of a person.\n\n\"Gel\n\n## Reprocess or delete a run [#reprocess-and-delete]\n\nReprocessing runs server-side processing again. Use it after a file failed or stalled, or after an engineer fixes the code that reads a file type. The stored result doesn’t change until you reprocess.\n\n* On the run page, hover over a file and choose **Reprocess**.\n* To reprocess several files, select them in the files table and choose **Reprocess** in the bar.\n* To reprocess whole runs, select them in a run list and choose **Reprocess** in the bar.\n\nReprocessing sends only the original instrument files. Files Data Hub generated, such as videos, previews, and CSVs, have no **Reprocess** button and are rebuilt from the originals.\n\n**Delete** marks the run deleted without removing stored files. To find a deleted run again, turn on **Show deleted runs** under **Add filter**, then open it and choose **Restore**. Anyone signed in can delete or restore a run.\n\n### Request an upload [#request-upload]\n\nA file that the watcher found but hasn’t uploaded shows **Pending**. Choose **Upload** on that file, or select runs and choose **Upload**, to ask the watcher to send it. The watcher picks up the request within a minute. On instruments that upload automatically, this needs watcher 1.2.1 or later; see [Roll out watcher releases](/docs/watcher-releases#what-each-version-adds).\n\nFor instruments in [manual upload mode](/docs/concepts#upload-mode), every file waits for this approval. **Dismiss** tells the watcher not to upload a file. See [Manage the watcher fleet](/docs/manage-watchers#approving-manual-uploads).\n\n## Use the same data outside the UI [#programmatic-access]\n\nYou can also list and fetch the same runs and files over the [REST API](/docs/api) or the [Model Context Protocol (MCP) endpoint](/docs/mcp). REST needs a [personal access token](/docs/concepts#personal-access-token); scopes and creation are covered in [Issue and revoke tokens](/docs/manage-tokens). MCP clients sign in through the browser instead, and clients that support MCP Apps show the same interactive run report you see here.\n" + }, + { + "page": "instrument-preprocessing", + "title": "Instrument data preprocessing", + "description": "What Data Hub does to a raw file after upload, which instrument types have a processor, and what supporting a new instrument takes.", + "markdown": "# Instrument data preprocessing (/docs/instrument-preprocessing)\n\n\n\n**For:** Lab operators, admins, and engineers\n\nAfter a [watcher](/docs/concepts#watcher) uploads a raw file, Data Hub can run a **preprocessor** on it: code that reads the vendor’s file, pulls out settings like wavelength or imaging mode, and builds something you can look at in the browser. Nine instrument types have one today. This page covers what each produces, why a file sometimes gets no processing, and what adding a new instrument involves.\n\n\n Support for a new instrument isn’t a settings change. It takes a new Python module, a database migration, an edit in two repositories, and a Lambda redeploy. See [Add support for a new instrument](#add-support-for-a-new-instrument).\n\n\n## How a file reaches a processor [#how-preprocessing-works]\n\n1. A file arrives in raw storage, which triggers the Lambda. The bucket notifies on every upload, with no per-instrument filtering.\n2. The Lambda checks the filename against a **filename gate**, a cheap test that skips files no processor wants before making any API call.\n3. It reads the instrument, then looks up a processor by the instrument’s [type](#set-the-instrument-type). Unmapped types stop here.\n4. The processor reads the file, extracts metadata, writes processed files when it has any, and updates the run through the API.\n5. The file’s status moves from `processing` to `completed` or `failed`.\n\n### Why an upload sometimes produces nothing [#why-an-upload-produces-nothing]\n\nTwo silent no-ops explain a file that uploaded and then produced nothing. Neither marks the file failed, because nothing went wrong:\n\n* **The filename missed the gate.** Each type accepts a narrow pattern, listed in the table below. A TapeStation run only triggers on its PDFs, so the peak-table CSVs it writes are stored but don’t start processing on their own.\n* **The instrument has no type set, or a type with no processor.** Files upload and stay browsable; you get no metadata and no reports.\n\nFilename gates apply to uploads only. Choosing **Reprocess** in the web app skips the gate, because clicking it states your intent. That’s why a file that produced nothing on upload can still process when you reprocess it by hand.\n\n### Set the instrument type [#set-the-instrument-type]\n\nProcessing is chosen by the instrument’s **type**, not by its instrument ID. The ID (`azure-cielo-qpcr`) names one physical machine. The type (`qpcr`) names a vendor’s file format, and several instruments can share one.\n\nTo set it, open the instrument page as an admin, choose **Edit details** from the three-dot menu, and pick an **Instrument type**.\n\n\n One type means one vendor’s output format. Names like `qpcr` and `fplc` read as generic, but each is bound to a single vendor’s parser: `fplc` expects ÄKTA PDFs, `qpcr` expects Azure Cielo exports. Typing a different vendor’s instrument into an existing type feeds files to a parser that can’t read them. Supporting a second vendor means splitting the type, which is a code change.\n\n\n## Supported instrument types [#supported-instruments-at-a-glance]\n\n| Type | Vendor and model | Gate for uploads | You get |\n| -------------------- | --------------------------- | --------------------------- | --------------------------------------------------------------- |\n| `tape_station` | Agilent 4150 TapeStation | `.pdf` | Tape-type badge, PDF reports |\n| `fplc` | Cytiva ÄKTA FPLC | `.pdf` | PDF you can read on the run page, no metadata |\n| `gel_doc` | Azure 600 Gel Doc | `.tif`, `.tiff` | Contrast-enhanced PNG, imaging badges |\n| `qpcr` | Azure Cielo qPCR | `.csv`, `.pdf`, `.aze` | Dye-channel badges, melting curves, the instrument’s PDF report |\n| `epson_v700_scanner` | Epson V700 Scanner | `.tif`, `.tiff` | Plate preview JPG, colony measurements |\n| `hina_microscope` | Hina Microscope (Nikon ND2) | `.nd2` | Channel overlay JPG, channel badges |\n| `plate_reader` | SpectraMax iD3 and iD5 | `.xls` | Well-data CSV, interactive plate maps |\n| `dishcam` | DishCam time-lapse camera | `.tif`, `.tiff`, `run.json` | A video of each photo stack, camera-setting badges |\n| `aunty` | Unchained Labs Aunty | `.xlsx` | A plate with a curve chart in each well, experiment badges |\n\n## Agilent 4150 TapeStation [#agilent-4150-tapestation]\n\nReads the tape type out of peak-table CSV filenames and shows TapeStation reports on the run page.\n\n* **Expected files**: CSVs named like `YYYY-MM-DD - HH-MM-SS-_peakTable.csv`, plus matching PDFs\n* **Metadata**: `Tape Type`, for example `gDNA` or `HSD1000`, when the filename matches that pattern\n* **Processed files**: none\n* **On the run page**: tape-type badge, a PDF viewer, and peak-table reports\n\nUploads trigger on the PDF, not the CSV. A run of CSVs alone stays unprocessed until you reprocess it by hand.\n\n## Cytiva ÄKTA FPLC [#akta-fplc]\n\nStores ÄKTA chromatography PDFs so you can read them in the browser. The processor reads nothing out of the PDF itself yet, so this type gets no badges or filters.\n\n* **Expected files**: `.pdf` reports from the ÄKTA\n* **Metadata**: none\n* **Processed files**: none\n* **On the run page**: files table, plus each PDF embedded under Report Data\n\nA newly seeded FPLC stays `generic` on purpose until an operator has checked its output format.\n\n## Azure 600 Gel Doc [#azure-600-gel-doc]\n\nRebuilds each gel TIFF as a viewable image and reads the imaging settings the instrument recorded.\n\nThe processor stretches intensities to the 1st to 99th percentile, per RGB channel or as grayscale, then lays every page of the TIFF side by side in one 300 dpi PNG. The stretch is cosmetic: it makes faint bands visible on screen and doesn’t change the raw file.\n\n* **Expected files**: TIFF from the Azure 600\n* **Metadata**: `capture_type`, `imaging_mode` (true color, fluorescence, or chemiluminescence), `wavelengths`, `colors`, read from the TIFF `XPComment` tag\n* **Processed files**: contrast-enhanced PNG\n* **On the run page**: imaging badges, files table, image viewer for the PNGs\n\nThose metadata fields also drive the column filters on the instrument page.\n\n## Azure Cielo qPCR [#azure-cielo-qpcr]\n\nReads dye channel names, melting curves, and the instrument’s PDF report.\n\n* **Expected files**:\n * the Cq Values CSV, for dye channels\n * the melting curve CSV (a name ending in `_MeltingCurve.csv`)\n * the PDF report the instrument writes\n * `.AZE` project files\n* **Metadata**: `dye_channels`, the unique names in the Cq Values CSV’s `Fluorescence` column\n* **Processed files**: `{run_id}_melting_curve_derivatives.csv` and `{run_id}_melting_curve_plate.json` from the melting curve CSV. An `.AZE` file produces `{run_id}_aze_melting_curve_derivatives.csv`, `{run_id}_aze_melting_curve_plate.json`, and `{run_id}_aze_experiment.json`.\n* **On the run page**: dye-channel badges, the PDF report under **Report Data**, and a **Melting Curves** section\n\nThe **Melting Curves** section shows the plate for one dye channel at a time. Click a well to see its melting curve and derivative, with a dashed line at the melting temperature. A run with more than one PDF gets previous and next buttons under **Report Data**.\n\nThe other CSV files a qPCR run writes upload normally and show as completed, but contribute no metadata.\n\n### `.AZE` project files [#aze-project-files]\n\nData Hub reads melting curves from the Azure Cielo’s native `.AZE` project files as well as from the CSV export. Melting temperatures match the CSV export, but other values can differ. That’s because `.AZE` files store raw fluorescence, while the CSV export can be adjusted for baseline. A project with no melt data completes with no processed files.\n\nThe watcher uploads `.AZE` files only when `*.AZE` is in the instrument’s file patterns. File patterns live in the watcher’s config on the instrument PC, so add it there with `data-hub-watcher config edit` and restart the watcher. Patterns are case-sensitive on some systems, so match the case the instrument uses. See [Watcher configuration](/docs/configuration-reference#field-reference).\n\n## Epson V700 Scanner [#epson-v700-scanner]\n\nFinds agar plates in a flatbed scan and measures the colonies on them.\n\nThe processor looks for plates sitting in gold 3D-printed frames. For each plate it finds, it crops the plate, then detects colonies: crop the edge, compare against the plate background, apply a difference-of-Gaussians filter, threshold, and measure whatever survives. Measurements come out in physical units, so colony areas are comparable between scans.\n\n* **Expected files**: TIFF plate scans\n* **Metadata**: `dpi`, `color_mode`, `plate_count`, `plate_boxes`, plus `colony_detection` summaries when colonies were measured\n* **Processed files**: a web-sized JPG with plate outlines and colony boxes drawn on, and a `_colonies.csv` of per-colony area, centroid, eccentricity, and mean RGB\n* **On the run page**: scanner badges, files table, and reports for the processed CSV\n\nWhen no plates are found, you still get a resized JPG of the whole scan, and colony detection is skipped. An empty `plate_count` usually means the frames weren’t recognized rather than that the plates were blank.\n\n## Hina Microscope [#hina-microscope]\n\nTurns each Nikon ND2 into a single composite image you can view in the browser.\n\nEvery channel is reduced to one 2D frame (a maximum projection over Z, the first index over time or position), stretched to the 1st to 99th percentile, then composited onto a brightfield background in each channel’s fluorophore color. The processor reads one frame at a time, so large stacks process without loading the whole file.\n\n* **Expected files**: ND2\n* **Metadata**: `sizes`, `channels` (name, excitation and emission wavelength, color), and `dimensions`\n* **Processed files**: an RGB overlay JPG for every file in the run\n* **On the run page**: channel and size badges, files table, image viewer for the JPGs\n\nChannel layout is identical across a Hina run, so metadata is written once by whichever file finishes processing first. Later files still get their JPG and skip the metadata write. Reprocessing one file therefore won’t correct run metadata if the layout changed mid-run.\n\nAn image too large to preview at full size gets a smaller preview instead. For example, a 6221 × 6221 pixel image is previewed at 3110 × 3110 pixels. The raw file is unchanged, and you can download it at full size. A file that showed **Failed** for this reason earlier gets its preview when you choose **Reprocess** on the file or on the run.\n\nTwo kinds of image still show **Failed** instead of a preview: an image from a color camera, which Data Hub can’t preview yet, and an image far larger than the Hina Microscope has produced so far. The message on the file says which. In both cases the raw file is stored and you can download it.\n\n## SpectraMax iD3 and iD5 plate readers [#spectramax-id3-and-id5-plate-readers]\n\nBoth models share one processor, which converts SoftMax Pro exports into the well data behind the interactive plate maps.\n\n* **Expected files**: SoftMax `.xls`, which is UTF-16 text with a `Plate:` header rather than a real Excel workbook\n* **Metadata**: `measurement_mode` (Absorbance, Fluorescence, or Luminescence), `measurement_type` (Endpoint, Kinetic, Spectrum, or Well Scan), and `wavelengths` as bare numbers with no unit suffix\n* **Processed files**: `{run_id}_raw_well_data.csv`\n* **On the run page**: wavelength, mode, and type badges, plus plate maps built from that CSV\n\nWhat the plate map shows depends on the read:\n\n* **Kinetic** reads get a time slider. Reads that skip the edge wells keep their time values, and runs that SoftMax Pro stopped early still process.\n* **Spectrum** reads (wavelength scans) get a wavelength slider.\n* **Luminescence** reads show no wavelength, because luminescence doesn’t use one.\n\nSoftMax Pro writes `#SAT` in a saturated well and `#Low` when the flash lamp missed a well. Those wells show as blank on the plate map, and the rest of the file processes normally.\n\n## DishCam [#dishcam]\n\nTurns each DishCam photo stack into a video you can play on the run page.\n\n* **Expected files**: multi-page TIFF photo stacks, plus the `run.json` file the camera writes next to them\n* **Metadata**: camera settings from `run.json`: `fps`, `measured_fps`, `frames`, `duration_seconds`, `quality`, `format`, `started`, and `finished`\n* **Processed files**: an MP4 video and a JPEG still frame for each stack\n* **On the run page**: a video player at the top of the report, with play, pause, and a timeline you can drag. The still frame shows before you press play.\n\nThe video plays at the capture rate or at 10 photos per second, whichever is faster, so slow captures stay watchable. The stored `fps` metadata still records the real capture rate.\n\nA stack processes once both the stack and its `run.json` are in storage, in either order. When a run gathers captures from several folders, each stack uses the `run.json` from its own folder. Watcher 1.1.0 and later keep every folder’s `run.json`, renaming later copies with a short tag such as `run~3f9a1c2b.json`. See [Roll out watcher releases](/docs/watcher-releases#what-each-version-adds).\n\nLarge stacks are read one photo at a time. If a stack can’t finish within the [15-minute processing limit](#when-processing-fails), it shows **Failed**, and the message says how far through the photos it got. You can still download the raw file.\n\n## Unchained Labs Aunty [#unchained-labs-aunty]\n\nReads Aunty exports, which measure protein melting temperature and particle size, and draws a plate of curves.\n\n* **Expected files**: the `.xlsx` export from the Aunty software. Thermal ramps, sizing runs, isothermal holds, and exports that contain only the results table without curves all process.\n* **Metadata**: `experiment_type` (`thermal_ramp`, `sizing`, or `isothermal`), `experiment_count`, and the first experiment’s settings: `analysis_mode`, `start_temp_c`, `end_temp_c`, `rate_c_per_min`, `temperature_c`, `analysis_version`, and `experiment_name`\n* **Processed files**: `{run_id}_aunty_curves.csv`, with every curve point for every well, and `{run_id}_aunty_plate.json`, which drives the plate view\n* **On the run page**: a plate with a small chart in each well. Click a well to open its full chart.\n\nOn the instrument page, filter Aunty runs by experiment type, analysis mode, temperature, and ramp rate.\n\n## Types with no processor [#instruments-without-a-matching-processor]\n\n`generic` and `instant_raman` have no processor, and neither does an instrument whose type is unset. Those uploads still work: files are stored, and you can browse, search, claim, comment on, and download them. What you don’t get is extracted metadata, badges, filters, or processed artifacts.\n\nLeaving a new instrument as `generic` is the safe default. It keeps data flowing while someone confirms which processor, if any, can read its files.\n\n## When processing fails [#when-processing-fails]\n\nA file with status `failed` carries an error message on the run page. Open the run, then choose **Reprocess** on that file, or select runs in a run list and choose **Reprocess**. See [Reprocess or delete a run](/docs/browse-runs#reprocess-and-delete).\n\nProcessing one file has a 15-minute limit. A file that hits the limit, or is too large to process at all, shows **Failed** with a message that explains why. The raw file stays in storage, and you can download it.\n\nA file that stays in processing for more than 20 minutes without finishing shows **Stalled**. That usually means the processor stopped without reporting back. Reprocess it to try again.\n\nReprocessing is also the fix after an engineer corrects a parser: the code changed, but the stored result didn’t, until you ask for it again.\n\n## Add support for a new instrument [#add-support-for-a-new-instrument]\n\n**For:** Engineers\n\nAdding an instrument type is a development task spanning the web app and the Lambda, not a configuration change an admin can make in the dashboard. Budget for a database migration and a Lambda image deploy.\n\n\n \n **Add the type.** Extend `instrumentTypeEnum` in `web/lib/db/schema.ts` and generate an `ALTER TYPE … ADD VALUE` migration. Reuse an existing type only when the new instrument writes the same vendor’s format.\n \n\n \n **Write the processor.** Add a module under `lambda/src/data_hub_lambda/` exposing `process_file(instrument_id, run_id, filename)`, which reports progress through the Data Hub API.\n \n\n \n **Register it twice.** Add the type to `PROCESSORS` in `lambda/src/data_hub_lambda/processors.py`, with its filename gate, and to `PROCESSABLE_INSTRUMENT_TYPES` in `web/lib/instruments/processable-types.ts`. These two lists have to agree: the Lambda uses the first to dispatch, and the web app uses the second to decide whether to offer **Reprocess**.\n \n\n \n **Add tests** for the processor and for the registry gate.\n \n\n \n **Deploy the Lambda image.** No new S3 trigger is needed, since the raw bucket already notifies on every upload.\n \n\n\nYou can exercise a processor locally without S3 or API access using the `data-hub-process` CLI, and drive the whole handler against a local S3 mirror with its `handler` subcommand.\n\nFull detail, including the archive builder and the Docker build, is in [the Lambda developer docs](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/lambda.md#adding-a-new-instrument--processor).\n" + }, + { + "page": "configure-notifications", + "title": "Configure notifications", + "description": "Choose in-app and Slack notifications for new runs, comments, service messages, and feedback, including per-instrument subscriptions and personal Slack direct messages.", + "markdown": "# Configure notifications (/docs/configure-notifications)\n\n\n\n\n\n**For:** Everyone\n\nData Hub can notify you in the app and in Slack when new runs arrive, when someone comments on a run you care about, when a script sends you a message, and when feedback you sent is closed. Open **Settings → Notifications** while signed in. Everything on that page is a personal choice that affects only you.\n\n\"Settings\n\nConnections that the whole workspace shares, such as the Slack channel for new runs and the Slack app behind direct messages, are under **Settings → Integrations** and only admins can change them. See [Set up integrations](/docs/integrations).\n\nIn-app and Slack preferences are independent: you can turn on either channel for a type without turning on the other.\n\n## In-app preferences [#in-app-preferences]\n\n| Setting | Effect |\n| ------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |\n| **Mute all instrument notifications** | Suppresses new-run notifications for every instrument. Per-instrument choices are preserved and resume when you unmute. |\n| **Comments on runs you ran** | Notify when someone comments on a run you are attributed to. |\n| **Replies to your comments** | Notify when someone comments on a run you previously commented on. |\n| **Service notifications** | Show messages that integrations and scripts send you through the API. |\n| **Updates on your feedback** | Notify when a [feedback report](/docs/send-feedback) you sent is resolved or declined. |\n| **New feedback** | Admins only. Notify when someone sends feedback about Data Hub. |\n\n### Per-instrument subscriptions [#per-instrument-subscriptions]\n\nUnder **Per-instrument**, toggle the instruments you want new-run notifications for. Rows are disabled while **Mute all** is on. You can also subscribe from the notifications switch on an instrument’s page, or from the instruments table’s notifications column when that column is shown.\n\n## The notification bell [#notification-bell]\n\nThe bell icon in the app lists recent notifications:\n\n* New runs are grouped by day and instrument, and each group expands to show its runs.\n* Clicking a comment notification opens the run and scrolls to that comment.\n* Messages sent by a script through an API token show the token’s name with a key icon.\n\n## Slack direct messages [#slack-direct-messages]\n\nConnect your Slack account on the same settings page to receive matching events as personal direct messages. After connecting, choose which types also go to Slack:\n\n* New instrument runs (respects your instrument subscriptions and mute)\n* Comments on runs you ran\n* Replies to your comments\n* Service notifications\n* Updates on your feedback\n* New feedback (admins only)\n\nDisconnect or reconnect from the same card. You have to connect with your organization’s Slack workspace; Data Hub rejects any other workspace.\n\nIf **Connect to Slack** is missing or doesn’t finish, an admin hasn’t set up the Slack app yet. See [Slack app for direct messages](/docs/integrations#slack-app).\n\n## Workspace Slack channel [#workspace-slack-channel-admins]\n\nThe shared Slack channel that gets a message whenever a new run is created is set by admins under **Settings → Integrations**. See [Slack channel for new runs](/docs/integrations#slack-channel).\n\n## Set up the Slack app for personal DMs [#set-up-the-slack-app-for-personal-dms]\n\nThe Slack app setup, including scopes, redirect URLs, and where to enter its credentials, is in [Slack app for direct messages](/docs/integrations#slack-app).\n" + }, + { + "page": "send-feedback", + "title": "Send feedback and see what’s new", + "description": "Report a bug or request a feature about Data Hub from the app or an AI assistant, follow the report’s status, and read the in-app changelog.", + "markdown": "# Send feedback and see what’s new (/docs/send-feedback)\n\n\n\n**For:** Everyone\n\nUse **Feedback** to tell the team that runs Data Hub about a bug, a missing feature, or anything else about the app itself. Each report becomes an issue in your workspace’s Linear, and Data Hub tells you when that issue is closed.\n\nFeedback is for problems with Data Hub. A problem with one run’s data, such as a wrong plate map, belongs in a comment on that run, where the people who ran it will see it.\n\n## Send a report [#send-a-report]\n\n\n \n Open the account menu at the bottom of the sidebar and choose **Feedback**.\n \n\n \n Pick a **Type**: **Bug**, **Feature request**, or **Other**.\n \n\n \n Enter a **Title** and a **Description**. Say what you expected and what happened instead.\n \n\n \n Optionally open **More details** and fill in **Trying to do**, **Tool**, and **Error message**.\n \n\n \n Choose **Send feedback**.\n \n\n\nData Hub also records the page you were on, so you don’t need to paste a link. If you send a report with the same title within 24 hours, Data Hub returns the report you already sent instead of creating a second one, unless that report has been closed.\n\n\n The **Feedback** item appears only after an admin connects Data Hub to Linear. If you don’t see it, ask an admin to follow [Set up integrations](/docs/integrations#linear).\n\n\n## What happens to a report [#what-happens-to-a-report]\n\n* Data Hub creates an issue in the Linear team your admins chose, with the label they picked for that type of report, if any.\n* The team works on the issue in Linear.\n* When the issue closes, Data Hub sends you a notification, and a Slack message if you turned that on.\n\nA report’s status follows its Linear issue:\n\n| Linear issue | Report status |\n| ---------------------------------- | ------------- |\n| Completed, such as **Done** | Resolved |\n| Canceled, or closed as a duplicate | Declined |\n| Anything else | Open |\n\nThe notification names the Linear status too, such as **Resolved (Done)** or **Declined (Duplicate)**. Turn it on or off with **Updates on your feedback** under **Settings → Notifications**; see [Configure notifications](/docs/configure-notifications#in-app-preferences).\n\nYou hear about a report once each time it closes. Moving the issue from one closed status to another doesn’t notify you again, but reopening and closing it does.\n\n## Send feedback from an AI assistant [#from-an-ai-assistant]\n\nAI assistants connected to Data Hub over [MCP](/docs/mcp) can send feedback for you, for example when a tool call fails. The assistant shows you the draft first and sends it only after you agree. It can also list the reports you sent and read one in full, by report ID or by Linear issue ID such as `ENG-1476`. The tools are listed under [MCP tools](/docs/mcp/tools).\n\nAPI tokens can’t send feedback, because every report needs a person behind it. Scripts that call `POST /api/v1/feedback` have to use a signed-in session.\n\n## Who sees which reports [#who-sees-which-reports]\n\nMembers see only the reports they sent. They can list them through the API or an AI assistant, without the Linear assignee, priority, or activity.\n\nAdmins review every report under **Settings → Feedback**:\n\n* **Open** and **Closed** are separate tabs.\n* Each row shows the Linear issue ID, status, label, priority, and assignee.\n* A report’s detail includes the Linear status changes and comments, and an **Open in Linear** link.\n\nStatus changes happen in Linear. Data Hub has no control for changing a report’s status. Admins can also turn on **New feedback** under **Settings → Notifications** to hear when someone sends a report.\n\n## See what’s new in Data Hub [#changelog]\n\nChoose **Changelog** in the account menu to see what has changed in Data Hub, newest first. It opens on top of the page you’re on. A dot on your account button means there’s an entry you haven’t opened yet. Many entries link to the page on this site that covers the change.\n" + }, + { + "page": "install-the-watcher", + "title": "Install and update the watcher", + "description": "Install, reinstall, verify, and manually update the Data Hub watcher on an instrument PC across operating systems.", + "markdown": "# Install and update the watcher (/docs/install-the-watcher)\n\n\n\n**For:** Lab operators\n\nInstall the watcher on an instrument PC, verify it, and apply manual updates. Fleet auto-update (what admins advertise and how PCs upgrade in the background) is covered in [Roll out watcher releases](/docs/watcher-releases).\n\n## Prerequisites [#prerequisites]\n\n* **[uv](https://docs.astral.sh/uv/getting-started/installation/)**: recommended. It installs Python for you; you do not need a separate Python install.\n* **A personal access token** starting with `dhub_`. Admins create these at **Settings → Access Tokens**; members cannot mint their own. See [Issue and revoke tokens](/docs/manage-tokens).\n* **Watch directory** and **file patterns** for the instrument output folder.\n\n## Install [#install]\n\nMost lab instrument PCs run Windows, so install the `windows-service` extra by default. It adds `pywin32`, which the watcher needs for the [Run as a Windows service](/docs/windows-service) setup:\n\n```sh\nuv tool install \"data-hub-watcher[windows-service]\"\n```\n\nThis puts the `data-hub-watcher` CLI on PATH in an isolated venv managed by `uv`.\n\nOn macOS or Linux, or on a PC you will only run in the foreground, omit the extra:\n\n```sh\nuv tool install data-hub-watcher\n```\n\n## Configure [#configure]\n\n```sh\ndata-hub-watcher init\n```\n\nThe wizard walks through environment, API key, instrument, watch directory, file patterns, run detection, stability period, and upload mode. New instruments start as `pending` until an admin confirms them; see [Set up an instrument](/docs/set-up-an-instrument).\n\nConfig saves to `~/.data-hub/config.yaml` and syncs to the server. Intent-level guidance for detection and uploads is in [Configure run detection and uploads](/docs/configure-run-detection); every field is in [Watcher configuration](/docs/configuration-reference).\n\n## Verify [#verify]\n\n```sh\ndata-hub-watcher watch --dry-run\n```\n\nDry-run validates config, checks API reachability and instrument status, and previews matching files without starting the monitor.\n\n## Run [#run]\n\nOn a lab PC, run the watcher as a Windows service so it starts on boot, survives logout, and auto-updates in the background. From an **Administrator** terminal:\n\n```sh\ndata-hub-watcher service install\ndata-hub-watcher service start\n```\n\nLifecycle commands, log locations, and recovery settings are in [Run as a Windows service](/docs/windows-service).\n\nFor a one-off check, or on macOS or Linux, run in the foreground instead:\n\n```sh\ndata-hub-watcher watch\n```\n\nThe watcher monitors the directory, waits for files to stabilize, groups them into runs, uploads (or queues in manual mode), and heartbeats every 60 seconds. Stop a foreground watcher with `Ctrl+C`; stop the service with `data-hub-watcher service stop`.\n\n## Reinstall [#reinstall]\n\nReinstall the same package (for example after a broken venv or missing service extra):\n\n```sh\nuv tool install --reinstall data-hub-watcher\n# or with the Windows service extra:\nuv tool install --reinstall \"data-hub-watcher[windows-service]\"\n```\n\nOn Windows services, follow with `data-hub-watcher service reinstall` from an Administrator shell so the service and upgrade Scheduled Task point at the new install.\n\n## Change configuration later [#change-configuration-later]\n\n```sh\ndata-hub-watcher config edit # re-prompt each field\ndata-hub-watcher config open # open the YAML in your editor\ndata-hub-watcher config show # print the current config\n```\n\nChanges sync to the server automatically.\n\n## Manual update (`self-update`) [#manual-update-self-update]\n\nUse this when you want an upgrade immediately, or on a non-service install. Background fleet upgrades are described in [Roll out watcher releases](/docs/watcher-releases).\n\n```sh\ndata-hub-watcher self-update # check + upgrade if needed\ndata-hub-watcher self-update --check # report status only\ndata-hub-watcher self-update --force # re-run even if versions match\n```\n\nThe command asks the API for the latest published version and upgrades according to how the watcher was installed:\n\n| Install method | Upgrade flow |\n| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| Windows + `uv tool install` | Routes through the `DataHubWatcherUpgrade` Scheduled Task. The CLI returns when the task accepts; the service restarts automatically. Tail `~/.data-hub/upgrade-worker.log` for progress. |\n| POSIX + `uv tool install` | `uv tool install --reinstall data-hub-watcher==` inline. **Restart the watcher** afterward. |\n| Plain venv `pip install` | `pip install -U data-hub-watcher==` inline. **Restart the watcher.** |\n| Editable / `uv sync` checkout | Refused; upgrade with `git pull && uv sync`. |\n\n\n On Windows uv-tool installs, `data-hub-watcher service install` (or `service reinstall`) must have run once from an Administrator shell so the upgrade Scheduled Task exists.\n\n\n### Pin a specific version [#pin-a-specific-version]\n\n```sh\nuv tool install data-hub-watcher==1.2.1\n```\n\nSome features need a minimum watcher version, such as uploading files larger than 5 GB (1.2.0) or **Request upload** on instruments that upload automatically (1.2.1). Check [What each version adds](/docs/watcher-releases#what-each-version-adds) before pinning an older version.\n\nRun `self-update --check` to confirm the server’s advertised target. When the server’s latest moves past your pin, the next auto-update tick tries to upgrade again. Fleet-wide pins and rollbacks are admin actions under [Roll out watcher releases](/docs/watcher-releases).\n" + }, + { + "page": "windows-service", + "title": "Run as a Windows service", + "description": "Install and operate the Data Hub watcher as a Windows service so it starts on boot, survives logouts, and can auto-update in the background.", + "markdown": "# Run as a Windows service (/docs/windows-service)\n\n\n\n**For:** Lab operators\n\nOn a lab instrument PC, run the watcher as a Windows service. This is the default setup: the service starts on boot, keeps running after you log out, restarts on failure, and participates in [fleet auto-update](/docs/watcher-releases). Foreground `data-hub-watcher watch` is for quick checks and non-Windows hosts.\n\n## Install the service extra [#install-the-service-extra]\n\nThe service depends on `pywin32`, which ships in the `windows-service` extra. The [default install](/docs/install-the-watcher#install) already includes it. If you installed without it, add it now:\n\n```sh\nuv tool install \"data-hub-watcher[windows-service]\"\n```\n\nRun [`data-hub-watcher init`](/docs/install-the-watcher#configure) first if you have not configured the watcher yet. The service uses the same `~/.data-hub/config.yaml` and per-environment `.env` file.\n\n## Install and start [#install-and-start]\n\nFrom an **Administrator** terminal:\n\n```sh\ndata-hub-watcher service install\ndata-hub-watcher service start\n```\n\n`service install` registers delayed auto-start (so it does not race the network stack at boot), configures restart-on-failure recovery, and registers the `DataHubWatcherUpgrade` Scheduled Task used by the auto-updater.\n\n\n The Windows auto-update path requires that `service install` (or `service reinstall`) has been run from an Administrator shell at least once. Fleet PCs upgrading into an auto-update-capable build for the first time need a one-time `service reinstall` to pick up the task.\n\n\n## Lifecycle commands [#lifecycle-commands]\n\n| Command | What it does |\n| ------------------------------------ | --------------------------------------------- |\n| `data-hub-watcher service install` | Register the service |\n| `data-hub-watcher service start` | Start the service |\n| `data-hub-watcher service stop` | Stop the service |\n| `data-hub-watcher service status` | Show whether the service is running |\n| `data-hub-watcher service uninstall` | Remove the service |\n| `data-hub-watcher service reinstall` | Stop, uninstall, install, and start in one go |\n\n`service reinstall` is the right command after an out-of-band wheel swap (for example a manual `uv tool install -U` from an Administrator shell). Stop and uninstall are best-effort, so it works even if the service is already gone.\n\n`service install` and `service reinstall` accept `--env-path PATH` to override which `.env` file the service loads (defaults to `~/.data-hub/.env.`).\n\n## Do not run `watch` alongside the service [#do-not-run-watch-alongside-the-service]\n\nRunning `data-hub-watcher watch` interactively while the service is also running is not supported: both processes would race on the same log file and watch directory. Stop the service first if you need the CLI for debugging:\n\n```powershell\ndata-hub-watcher service stop\ndata-hub-watcher --verbose watch\ndata-hub-watcher service start\n```\n\n## Where the service writes logs [#where-the-service-writes-logs]\n\nThe service and the CLI share one rotating log:\n\n* `C:\\ProgramData\\DataHubWatcher\\watcher.log`\n\nA separate `service-bootstrap.log` next to it captures crashes before the service control dispatcher takes over (missing `pywin32`, moved venv, corrupt install). To turn on debug logging without a reinstall, add `DATA_HUB_WATCHER_LOG_LEVEL=DEBUG` to the env file the service loads and restart the service.\n" + }, + { + "page": "configure-run-detection", + "title": "Configure run detection and uploads", + "description": "Choose how the watcher groups files into runs, when files are ready to upload, and whether uploads are automatic or approved.", + "markdown": "# Configure run detection and uploads (/docs/configure-run-detection)\n\n\n\n**For:** Lab operators\n\nTune three behaviors that decide what becomes a run and when bytes leave the instrument PC: run detection, stability period, and upload mode. Field-by-field schema lives in [Watcher configuration](/docs/configuration-reference); this page is the intent guide.\n\nChange these during `data-hub-watcher init`, with `data-hub-watcher config edit`, or by editing `~/.data-hub/config.yaml` then syncing.\n\n## Group files into runs [#group-files-into-runs]\n\nA **run** is one acquisition: a logical group of files the watcher reports together. **Run detection** maps each file’s path (relative to the watch directory) to a run ID.\n\nCommon choices:\n\n* **Prefix from filename**: a regex capture, for example everything before the first underscore so `RUN001_data.csv` → `RUN001`\n* **One subdirectory per run**: each folder under the watch directory is its own run, so `RUN001/data.csv` → `RUN001`\n\nPresets and custom regex options are listed under [Run detection](/docs/configuration-reference#run-detection). Set `recursive: true` when files live in subfolders; otherwise only the top level is watched.\n\nWhen a run gathers files from several folders, two files can share a name, such as a `run.json` in each capture folder. Watcher 1.1.0 and later keep every copy and add a short tag to the later names, such as `sample~3f9a1c2b.tif`. Older watchers kept only the first; see [Files with the same name were skipped](/docs/troubleshoot-a-watcher#files-with-the-same-name-were-skipped).\n\nIf files appear on disk but never become runs, check for `pattern_mismatch` events on the watcher detail page and run `data-hub-watcher watch --dry-run`.\n\n## Wait until writes finish [#wait-until-writes-finish]\n\n**Stability period** is how many seconds a file must keep the same size and modification time before the watcher treats it as fully written. Default is 5 seconds (allowed range 1–300). Raise it for instruments that write large files slowly.\n\nA file that keeps changing past `max_stability_wait_seconds` (default 300, or 5 minutes) is abandoned with a `stability_timeout` error event. Raise that cap for instruments whose writes take longer; it must be at least the stability period.\n\n## Choose auto or manual upload [#choose-auto-or-manual-upload]\n\n| Mode | Behavior |\n| ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |\n| **auto** | Upload as soon as a file is detected and stable. With watcher 1.2.1 and later, the watcher also picks up files someone requests with **Upload** on the run page, and retries uploads that a restart cut off. |\n| **manual** | Report the run, but wait for server approval before uploading bytes. The watcher checks the upload queue once a minute. |\n\nUse manual mode when someone must approve which files leave the instrument PC. Admins manage the queue from the dashboard; see [Manage the watcher fleet](/docs/manage-watchers#approving-manual-uploads).\n\n## Existing files on first setup [#existing-files-on-first-setup]\n\n`initial_scan` controls the backlog already in the watch directory when an environment is first entered:\n\n* **`production` default**: upload the backlog (`full`)\n* **`staging` / `preview` default**: record a baseline and skip existing files (`new-only`)\n\nOverride in config when you need the other behavior. Details: [Initial scan](/docs/configuration-reference#initial-scan).\n" + }, + { + "page": "manage-watchers", + "title": "Manage the watcher fleet", + "description": "Monitor watcher health, deregister watchers, and approve or request uploads from the admin dashboard.", + "markdown": "# Manage the watcher fleet (/docs/manage-watchers)\n\n\n\n**For:** Administrators\n\nThe **Watchers** area is the fleet view: every registered watcher, its health, deregistration, and the upload queue. Cutting releases and advertising versions is separate: [Roll out watcher releases](/docs/watcher-releases).\n\n## Monitor watcher health [#monitor-watcher-health]\n\nEach watcher sends a [heartbeat](/docs/concepts#heartbeat) every 60 seconds with version, instrument, watch directory, upload mode, activity counters, and uptime. The fleet list surfaces:\n\n* **Status**: **Online**, **Unresponsive** (no heartbeat for more than 5 minutes), **Stopped** (shut down cleanly), or **Registered** (no first heartbeat yet). See [Watcher status](/docs/concepts#watcher-status).\n* **Last heartbeat**: a stale value is the first sign of a stopped, disconnected, or crashed watcher\n* **Hostname and instrument assignment**\n* **Events**: per-watcher log (`watcher_started`, `run_reported`, `file_uploaded`, `update_*`, and `error` with a `details.kind` discriminator)\n\nEvents are the primary remote-diagnosis tool. For operator-side symptoms, see [Troubleshoot a watcher](/docs/troubleshoot-a-watcher).\n\n## Deregister a watcher [#deregister-a-watcher]\n\nEach instrument can have at most one active watcher. Deregister the old one when a PC is reimaged or replaced, or to clear “instrument already has an active watcher” on a new install.\n\n* **Web app**: open the watcher under **Watchers** and click **Deregister**, or choose **Deregister** from the watcher’s row menu\n* **API**: `DELETE /api/v1/watchers/:watcherId` (requires `watchers:admin`)\n\nDeregistration is a soft-delete: heartbeats, events, and runs stay under **Watchers → Deregistered**. A watcher removed through the API with a token shows that token’s name as the one that removed it.\n\nRetiring an instrument also deregisters its watchers automatically; see [Retire an instrument](/docs/retire-an-instrument).\n\nYou don’t need to deregister a watcher just because its token was revoked. Re-running `data-hub-watcher init` with a new token takes over the existing registration; see [After revoking a token](/docs/manage-tokens#after-revoking-a-token).\n\n## Request uploads [#request-uploads]\n\n**Upload** on a run page, or on selected runs in a run list, asks the watcher to send files that are still on the instrument PC. The watcher polls the server’s upload queue once a minute and uploads the files that were requested.\n\n* In **manual** upload mode, every file waits in this queue until someone requests it.\n* In **auto** upload mode, files upload on their own, and the queue covers files that need another try. This works with watcher 1.2.1 and later; older watchers in auto mode never pick up requests, so the file shows **Uploading** and stays that way.\n\n## Approving manual uploads [#approving-manual-uploads]\n\nFor instruments in [manual upload mode](/docs/concepts#upload-mode), the watcher reports runs but waits for server approval before uploading file bytes. Approve a file with **Upload**, or choose **Dismiss** to tell the watcher not to upload it.\n\nSafeguards that keep the queue healthy:\n\n* Changing an instrument’s `watch_directory` reverts pending upload requests to `detected` so stale paths drain instead of erroring\n* A queued file that keeps failing (missing on disk or failing to upload) is retried on at most three polls, then cancelled server-side\n" + }, + { + "page": "watcher-releases", + "title": "Roll out watcher releases", + "description": "Publish watcher versions to PyPI, advertise them per environment, set a minimum version, force mandatory updates, roll the fleet back, and see what each version adds.", + "markdown": "# Roll out watcher releases (/docs/watcher-releases)\n\n\n\n\n\n**For:** Administrators\n\nLab PCs upgrade themselves toward the version you advertise per environment. You control that from **Settings → Watchers**. How an individual PC installs or runs `self-update` is in [Install and update the watcher](/docs/install-the-watcher).\n\n## How fleet upgrades reach a PC [#how-fleet-upgrades-reach-a-pc]\n\nThe in-process updater runs from the heartbeat tick on every platform (not only Windows services). Roughly once an hour the watcher:\n\n1. Asks the server for the latest version and compares it to its own\n2. Upgrades only if a newer version is available, no files have uploaded for several heartbeats, and no run was reported recently (the activity-window guard). Releases flagged **mandatory** skip this guard\n3. Installs the new wheel and restarts into it (Windows services restart via failure-actions policy)\n4. Emits `update_started`, then `update_succeeded` or `update_failed`, visible on the dashboard\n\nAuto-update is disabled in the `preview` environment so pull-request preview deployments cannot push code to lab PCs.\n\n## Cut a new release [#cut-a-new-release]\n\nReleases are tag-driven and publish from a `production` commit.\n\n\n \n Bump the version in `watcher/pyproject.toml` and merge through `staging` to `production`.\n \n\n \n Tag and push from `production`:\n\n ```sh\n git checkout production && git pull\n git tag watcher-v1.3.0 && git push origin watcher-v1.3.0\n ```\n \n\n \n Approve the `pypi` deployment under **Actions → Publish watcher** in GitHub.\n \n\n \n In **Settings → Watchers**, pick the new version under **Latest version**. The list comes straight from PyPI, so a release shows up as soon as PyPI publishes it. If PyPI can’t be reached, the field accepts a typed version instead. Each environment has its own database, so set `staging` and `production` separately to roll the fleet gradually.\n\n \"Settings\n \n\n\n\n Always tag → publish → verify on PyPI → save. Saving **Latest version** before the wheel is live triggers a wave of `update_failed` events across the fleet.\n\n\n## Release settings [#release-config-fields]\n\n| Field | Purpose |\n| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| **Latest version** | The version watchers upgrade to. Leave it blank to pause self-updates. |\n| **Minimum supported version** | Optional. A watcher older than this has its heartbeats rejected with `426 Upgrade Required`, so it shows as unresponsive until it updates. It still checks for updates every hour, so it upgrades itself as long as **Latest version** is newer. Leave it blank for no minimum. |\n| **Mandatory update** | Skips the activity-window guard; see below. Has no effect while **Latest version** is blank. |\n\n## Mandatory updates [#mandatory-updates]\n\nToggling **Mandatory update** skips the activity-window guard and fires the upgrade on the next hourly check. The server still compares the running version to **Latest version**, so a correctly pinned PC is not forced past its pin.\n\nUse sparingly: a forced upgrade mid-acquisition can lose data. Reserve it for security fixes, wire-protocol breaks, or cases where leaving the bad version running is worse than restarting in flight.\n\n## Roll back [#roll-back]\n\nRollback is another release: set **Latest version** back to the older tag for the affected environment(s), turn **Mandatory update** on if you need to bypass the activity-window guard, and wait for the next hourly tick. Turn **Mandatory update** off once the fleet converges. The older wheel remains on PyPI; it is no longer advertised.\n\nIf **Minimum supported version** is set above the version you roll back to, lower it first. Otherwise the rolled-back watchers can’t check in.\n\n## What each version adds [#what-each-version-adds]\n\nSome Data Hub features need a minimum watcher version. Set **Latest version** to at least the version below to turn a feature on across the fleet.\n\n| Version | What it adds |\n| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| 1.2.1 | **Request upload** works on instruments that upload automatically; older watchers in auto mode leave the file showing **Uploading**. The watcher also retries an automatic upload that didn’t finish, such as one cut off by a restart or an update, the next time it starts. It tries on up to three starts. |\n| 1.2.0 | Files of 100 MB or more upload in 64 MB parts, and a failed part is retried on its own. Files larger than 5 GB can upload; older watchers are refused for files that large. |\n| 1.1.0 | When a run gathers files from several folders, files with the same name are all kept instead of only the first. Later copies get a short tag in the name, such as `sample~3f9a1c2b.tif`. Adds the `data-hub-watcher state forget` command. |\n\nAfter upgrading to 1.1.0 or later, files that an older watcher skipped because of a duplicate name aren’t uploaded on their own. To upload them, stop the watcher, run `data-hub-watcher state forget --prefix your_folder_name/` on the instrument PC, and start the watcher again. See [Troubleshoot a watcher](/docs/troubleshoot-a-watcher#files-with-the-same-name-were-skipped).\n\n## Operator-side failures [#operator-side-failures]\n\nIf `update_started` is followed by `update_failed`, or auto-update never fires, point operators at [Troubleshoot a watcher](/docs/troubleshoot-a-watcher#auto-update-never-fires) and [Install and update the watcher](/docs/install-the-watcher#manual-update-self-update).\n" + }, + { + "page": "troubleshoot-a-watcher", + "title": "Troubleshoot a watcher", + "description": "Fix watcher problems by symptom: will not start, files not detected, uploads stuck, auto-update quiet, and where to find logs.", + "markdown": "# Troubleshoot a watcher (/docs/troubleshoot-a-watcher)\n\n\n\n**For:** Lab operators and admins\n\nFind your symptom below. When you are stuck, the [logs](#where-are-the-logs) are the fastest way in. Admins can also inspect heartbeats and events under **Watchers** in the web app.\n\n## Setup and registration [#setup-and-registration]\n\n### “Instrument is still pending activation” [#instrument-is-still-pending-activation]\n\nThe instrument was registered but an admin has not confirmed it. Ask an admin to open **Instruments** and click **Confirm**. See [Manage instruments](/docs/manage-instruments).\n\n### “Instrument already has an active watcher” [#instrument-already-has-an-active-watcher]\n\nEach instrument can have at most one active watcher. An earlier install (often on another PC, or before a reimage) is still registered. The CLI prints the existing watcher id; deregister it before re-running `init`:\n\n* **Web app**: **Watchers**, open the existing watcher, click **Deregister**\n* **API**: `DELETE /api/v1/watchers/` with a bearer token that has `watchers:admin`\n\nDeregistration is a soft-delete: history stays under **Watchers → Deregistered**.\n\n### `init` rejects my API key [#init-rejects-my-api-key]\n\nThe wizard rejects values that do not start with `dhub_`, and strips zero-width and non-breaking-space characters that Windows clipboards inject. If a paste keeps failing, pass `--show-key` so you can see what the prompt received.\n\n## The watcher will not start [#the-watcher-will-not-start]\n\n* Run `data-hub-watcher config show` to confirm config and environment\n* Run `data-hub-watcher watch --dry-run`: it prints the first failing check\n* Confirm the instrument is `active`, not `pending`\n* On Windows, if the **service** will not start, see [Windows service](#windows-service)\n\n## The watcher shows Offline right after a restart [#the-watcher-shows-offline-right-after-a-restart]\n\nThe watcher checks in once a minute, so a watcher that was just restarted can take up to a minute to show as **Online**. Refresh the page after a minute. If it still shows **Offline** or **Unresponsive**, check that the process is running and see [“Connection error” or “Request timed out”](#connection-error-or-request-timed-out).\n\n## Heartbeats fail with `426 Upgrade Required` [#heartbeats-fail-with-426-upgrade-required]\n\nThe watcher is older than the **Minimum supported version** an admin set under **Settings → Watchers**, so the server rejects its heartbeats and it shows as unresponsive. It keeps checking for updates every hour and upgrades itself if **Latest version** is newer. To upgrade now, run `data-hub-watcher self-update`. See [Roll out watcher releases](/docs/watcher-releases#release-config-fields).\n\n## “Connection error” or “Request timed out” [#connection-error-or-request-timed-out]\n\nThe watcher cannot reach the Data Hub API. Check:\n\n* Network connectivity from the instrument PC\n* Environment in config (`staging`, `production`, or `preview`): `data-hub-watcher config show`\n* That the API base URL for that environment is reachable\n\n## Files are not being detected [#files-are-not-being-detected]\n\n* Confirm the watch directory and file patterns. The instrument’s page in the web app shows both, or run `data-hub-watcher config show` on the PC\n* Check that the instrument saves its files into that folder, not a shared network drive or another folder\n* Check file patterns. Globs are case-sensitive on some systems (`*.csv` does not match `data.CSV`)\n* If files live in subfolders, set run detection to recursive; see [Configure run detection and uploads](/docs/configure-run-detection)\n* Run `data-hub-watcher watch --dry-run` to preview matches\n* Look for `pattern_mismatch` events on the dashboard\n\n## Files are detected but not uploading [#files-are-detected-but-not-uploading]\n\n* In **manual** mode, files wait for approval in the upload queue; see [Manage the watcher fleet](/docs/manage-watchers#approving-manual-uploads)\n* Check the log for `upload_failed`\n* A revoked or expired token surfaces as `401 Unauthorized`. Create a new token and re-run `init`; see [Issue and revoke tokens](/docs/manage-tokens#after-revoking-a-token)\n\n### A requested upload stays on Uploading [#a-requested-upload-stays-on-uploading]\n\nOn an instrument in **auto** mode, **Request upload** needs watcher 1.2.1 or later. Older watchers in auto mode never read the upload queue, so the file shows **Uploading** and never arrives. Update the watcher; see [Roll out watcher releases](/docs/watcher-releases#what-each-version-adds).\n\n### Files larger than 5 GB don’t upload [#files-larger-than-5-gb-dont-upload]\n\nWatchers older than 1.2.0 can’t upload files larger than 5 GB, and Data Hub refuses the upload instead of letting it fail partway. Update the watcher to 1.2.0 or later, which uploads large files in parts.\n\n### An upload was cut off by a restart or an update [#an-upload-was-cut-off-by-a-restart-or-an-update]\n\nWatcher 1.2.1 and later retry an automatic upload that didn’t finish the next time the watcher starts, on up to three starts. Two events tell you when it stops trying:\n\n* `upload_retries_exhausted`: the upload still failed after three starts. Choose **Upload** on the file’s run page to try again.\n* `pending_upload_missing`: the file was no longer on disk when the watcher started, so there was nothing to retry.\n\nOlder watchers don’t retry; use **Upload** on the run page instead.\n\n### Files with the same name were skipped [#files-with-the-same-name-were-skipped]\n\nWhen a run gathers files from several folders, watchers older than 1.1.0 uploaded only the first file with a given name and skipped the rest. Watcher 1.1.0 and later keep every copy, adding a short tag to later names. To upload files an older watcher skipped:\n\n\n \n Update the watcher to 1.1.0 or later.\n \n\n \n Stop the watcher (`data-hub-watcher service stop`, or `Ctrl+C` in the foreground).\n \n\n \n Run `data-hub-watcher state forget --prefix your_folder_name/` on the instrument PC, where `your_folder_name/` is the folder under the watch directory that held the skipped files.\n \n\n \n Start the watcher again. It reports the files in that folder again and uploads the ones Data Hub doesn’t have.\n \n\n\n## Auto-update never fires [#auto-update-never-fires]\n\nCheck, in order:\n\n* Prefer a long-running service (or scheduled) install; a one-off console session may not stay up for the hourly tick\n* Environment is not `preview`: auto-update is hard-disabled there\n* Activity-window guard is not holding it up: run `data-hub-watcher self-update`, or ask an admin to flag the release **mandatory**\n* **Last Heartbeat** on the dashboard is recent; a stale watcher is not ticking\n\nUpgrade failure modes (`update_started` → `update_failed`, editable installs, missing Scheduled Task) are covered in [Install and update the watcher](/docs/install-the-watcher#manual-update-self-update) and [Roll out watcher releases](/docs/watcher-releases).\n\n## Windows service [#windows-service]\n\n### The service exits immediately [#the-service-exits-immediately]\n\nIf it stops before writing to `watcher.log`:\n\n* Check `C:\\ProgramData\\DataHubWatcher\\service-bootstrap.log` for a pre-dispatcher crash (missing `pywin32`, moved venv, corrupt install)\n* Query the Windows Application event log:\n\n```powershell\nGet-WinEvent -FilterHashtable @{LogName='Application'; StartTime=(Get-Date).AddHours(-1)} |\n Where-Object { $_.ProviderName -match 'Python|DataHubWatcher' -or $_.Message -match 'DataHubWatcher' } |\n Format-List TimeCreated, ProviderName, Id, LevelDisplayName, Message\n```\n\n* Run the service in the foreground to see the traceback:\n\n```powershell\n& \"C:\\path\\to\\venv\\Scripts\\python.exe\" -m win32serviceutil debug DataHubWatcher\n```\n\n### `self-update` says “Upgrade dispatched” but nothing happens [#self-update-says-upgrade-dispatched-but-nothing-happens]\n\nOn Windows uv-tool installs, the upgrade runs out of process under SYSTEM and takes 30–60s. Track it:\n\n```powershell\nGet-Service data-hub-watcher\nGet-Content ~/.data-hub/upgrade-worker.log -Tail 20 -Wait\n```\n\n### Turn on debug logging for the service [#turn-on-debug-logging-for-the-service]\n\nAdd `DATA_HUB_WATCHER_LOG_LEVEL=DEBUG` to the env file the service loads (typically `~/.data-hub/.env.`) and restart the service. No reinstall needed.\n\n## Where are the logs? [#where-are-the-logs]\n\nRotating logs (10 MB each, 5 backups). CLI `watch` and the Windows service share one file:\n\n| Platform | Log file |\n| ------------- | ------------------------------------------- |\n| Windows | `C:\\ProgramData\\DataHubWatcher\\watcher.log` |\n| macOS / Linux | `~/.data-hub/watcher.log` |\n\nAdd `--verbose` for debug console output: `data-hub-watcher --verbose watch`. Windows uv-tool upgrade output lands in `~/.data-hub/upgrade-worker.log`.\n\n## API clients get `401 Unauthorized` [#api-clients-get-401-unauthorized]\n\nThe bearer token is missing, mistyped, revoked, or expired. Check **Settings → Access Tokens** and re-issue if needed. See [Issue and revoke tokens](/docs/manage-tokens).\n\nA script that calls the watcher or processing routes with a browser session, instead of a token, also gets `401`. Those routes accept only personal access tokens; see [How a token acts](/docs/manage-tokens#how-a-token-acts).\n\n## MCP clients get `401 Unauthorized` [#mcp-clients-get-401-unauthorized]\n\nMCP doesn’t accept personal access tokens, so a client configured with `Authorization: Bearer dhub_…` gets a 401 on every call. Remove the header, leave only the URL, and let the client run its sign-in flow. Setup per client is in [MCP overview](/docs/mcp#connect-a-client).\n\nAn already-connected client that starts failing has usually lost its grant: sign in again from the client. If reads work and one tool fails with a missing-scope message, the client holds `read` but not `write`; reconnect and approve both.\n\n## Ask an agent to diagnose it [#ask-an-agent-to-diagnose-it]\n\nUse the `troubleshoot_instrument` MCP prompt under [MCP prompts](/docs/mcp/prompts).\n" + }, + { + "page": "administration", + "title": "Administration overview", + "description": "Admin entry point for instruments, watchers, tokens, members, and fleet releases.", + "markdown": "# Administration overview (/docs/administration)\n\n\n\n**For:** Administrators\n\nAdmins keep the workspace healthy: confirm new instruments, watch the fleet, issue tokens, manage who is an admin, roll out watcher releases, and connect Linear and Slack. Admins can also delete any run comment and review the feedback people send about Data Hub.\n\nThe admin-only pages under **Settings** are **Feedback**, **Integrations**, **Watchers**, and **Members**. Members don’t see them in the settings menu.\n\nRoles and scopes are defined in [Security and permissions](/docs/security). Day-to-day operator setup is in [Set up an instrument](/docs/set-up-an-instrument).\n\n## What you manage [#what-you-manage]\n\n\n \n\n \n\n \n\n \n\n \n\n \n\n \n\n \n\n \n\n" + }, + { + "page": "manage-instruments", + "title": "Manage instruments", + "description": "Confirm pending instruments, read an instrument’s watcher status, watched folder, and file patterns, and edit its name and type.", + "markdown": "# Manage instruments (/docs/manage-instruments)\n\n\n\n\n\n**For:** Administrators\n\nWhen a lab operator registers an instrument from the watcher, it appears as `pending` and cannot upload until you confirm it. Operator-side setup is in [Set up an instrument](/docs/set-up-an-instrument). Retiring and reactivating are in [Retire an instrument](/docs/retire-an-instrument).\n\n## Instrument states [#instrument-states]\n\nThe **Instruments** page has tabs for active, pending, and retired instruments.\n\n| Status | UI label | Meaning |\n| ---------- | -------- | ------------------------------------------------------------ |\n| `pending` | Pending | Registered but not confirmed. Watchers cannot upload. |\n| `active` | Active | Confirmed and accepting uploads. |\n| `inactive` | Retired | Deactivated; no new uploads. Historical runs stay browsable. |\n\n\"Instruments\n\n## Confirm a pending instrument [#confirm-a-pending-instrument]\n\n\n \n Open the Data Hub web app and go to **Instruments**.\n \n\n \n Open the **Pending** tab (or find the yellow `pending` badge).\n \n\n \n Click **Confirm**. Status becomes `active` and the operator’s watcher can upload.\n \n\n\nConfirming is admin-only for signed-in users. Under the hood it is `PATCH /api/v1/instruments/:id`. Token automation uses the `instruments:write` scope instead. See the [generated API docs](/docs/api).\n\n## Read an instrument’s page [#view-instrument-health]\n\nThe top of each instrument page shows what you need to tell whether its files will reach Data Hub:\n\n* **Watcher status**: a badge next to the instrument name. **Online** means the watcher checked in recently, **Offline** means it hasn’t, and **No Watcher** means none has registered. The watcher checks in once a minute, so a watcher that was just restarted can take up to a minute to show as online. Hover over the badge for details.\n* **Watched folder**: the folder the watcher reads from. Hover over it to see which PC it is on.\n* **File patterns**: the patterns the watcher uploads, such as `*.csv`.\n\nFiles saved anywhere other than the watched folder, such as a shared network drive, don’t appear in Data Hub. Neither do files whose names don’t match a pattern. The folder and patterns come from the watcher’s config on the instrument PC, so change them there with `data-hub-watcher config edit`; see [Watcher configuration](/docs/configuration-reference).\n\nWatcher health comes from heartbeats; see [Manage the watcher fleet](/docs/manage-watchers).\n\n## Edit an instrument [#edit-an-instrument]\n\nThe three-dot menu on an instrument page, and the row menu on the instruments table, hold the admin actions:\n\n* **Edit details**: change the **Display name** and the **Instrument type**\n* **View watcher**: open the watcher registered for this instrument\n* **Retire** or **Reactivate**: see [Retire an instrument](/docs/retire-an-instrument)\n\nThe instrument type decides which processor reads the instrument’s files, including DishCam and Aunty; see [Instrument data preprocessing](/docs/instrument-preprocessing). The instrument ID never changes, because it is the storage key prefix.\n" + }, + { + "page": "retire-an-instrument", + "title": "Retire an instrument", + "description": "Retire an instrument so it stops accepting uploads and watchers, while keeping historical runs and files.", + "markdown": "# Retire an instrument (/docs/retire-an-instrument)\n\n\n\n**For:** Administrators\n\nRetire an instrument when it is decommissioned or out of service for a long stretch. Retirement sets status to `inactive` (shown as **Retired**), removes it from the sidebar and dashboard lists of live instruments, and stops new uploads. Nothing is deleted from storage; you can reactivate later.\n\n## What retirement does [#what-retirement-does]\n\nWhen you retire an instrument:\n\n* Status becomes `inactive` (UI: **Retired**), with `retiredAt` / `retiredBy` audit fields\n* It stops accepting new runs and watchers\n* All existing runs and files stay browsable\n* Every active watcher on that instrument is deregistered in the same request so they stop heartbeating\n\nInstrument IDs stay permanent. Retire rather than trying to rename or delete an instrument that has already produced data.\n\n## Retire from the web app [#retire-from-the-web-app]\n\n\n \n Open the instrument’s detail page.\n \n\n \n Open the admin actions menu and choose **Retire**.\n \n\n \n Confirm in the dialog. The dialog lists run count and how many watchers will be deregistered.\n \n\n\nThe instrument moves to the **Retired** tab on **Instruments**.\n\n## Reactivate [#reactivate]\n\nFrom a retired instrument’s detail page, choose **Reactivate**. Status returns to `active` and the instrument reappears in the sidebar and dashboard.\n\nWatchers deregistered at retirement do **not** reconnect automatically. An operator must re-register (usually `data-hub-watcher init` on the instrument PC) after reactivation.\n\n## Via the API [#via-the-api]\n\n```sh\n# Retire\ncurl -X PATCH https://datahub.example.com/api/v1/instruments/bio-rad-cfx96 \\\n -H \"Authorization: Bearer dhub_your_token_here\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"status\":\"inactive\"}'\n\n# Reactivate\ncurl -X PATCH https://datahub.example.com/api/v1/instruments/bio-rad-cfx96 \\\n -H \"Authorization: Bearer dhub_your_token_here\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\"status\":\"active\"}'\n```\n\nSession callers need the admin role; bearer tokens need `instruments:write`.\n" + }, + { + "page": "manage-tokens", + "title": "Issue and revoke tokens", + "description": "Create, use, and revoke personal access tokens that authenticate the watcher and other Data Hub API clients, and see what a token can and can’t do.", + "markdown": "# Issue and revoke tokens (/docs/manage-tokens)\n\n\n\n**For:** Administrators\n\nPersonal access tokens authenticate the watcher, the processing Lambda, and scripts that call the Data Hub API. Creating and revoking tokens requires the workspace admin role. Members can view the list at **Settings → Access Tokens** but cannot create or delete tokens. Roles and scopes: [Security and permissions](/docs/security).\n\n## How a token acts [#how-a-token-acts]\n\nA token acts as itself, under its own name. It doesn’t act as the admin who created it or as any other person.\n\n* Anything a token does is recorded under the token’s name: comments, deleting a run, retiring an instrument, removing a watcher, and notifications sent through the API.\n* Run pages, instrument pages, watcher pages, comments, and the notification bell show the token’s name with a key icon.\n* A token can post comments, and it can edit and delete only the comments it posted.\n\nSome actions need a signed-in person, so tokens can’t do them at all: claiming runs, sending feedback, and reviewing feedback. AI assistants aren’t affected, because they sign in as the person who approved them; see [MCP overview](/docs/mcp).\n\nThe reverse also holds: the API routes that the watcher and the Lambda call accept only tokens. A browser session gets `401 Unauthorized` on watcher registration and heartbeats, upload links, file updates, run creation and updates, and instrument creation.\n\n## Create a token [#create-a-token]\n\n### In the web app [#in-the-web-app]\n\n\n \n Sign in as an admin.\n \n\n \n Go to **Settings → Access Tokens**.\n \n\n \n Click **Create token**.\n \n\n \n Enter a **Name** that identifies the client (for example, `FPLC watcher, Lab 201`). This is the name people see wherever the token acts, so make it recognizable.\n \n\n \n Pick an **Expiration**: 30 days, 90 days (the default), 1 year, or no expiry.\n \n\n \n Under **Scopes**, click a preset, or click **Customize scopes** to pick individual scopes.\n \n\n \n Click **Create token**.\n \n\n\nThe plaintext token is shown once: copy it immediately. It starts with `dhub_` followed by a 64-character hex string.\n\n### Scope presets [#scope-presets]\n\nA preset selects the minimum scopes a common client needs:\n\n* **Read-only**: view instruments, runs, files, watchers, and archive jobs; change nothing\n* **Watcher**: what a watcher needs to register, report runs, and upload files\n* **Lambda**: what the processing Lambda needs to read instruments and runs, and write run and file results\n\n\n The **Lambda** preset includes `runs:read`, which DishCam processing needs to pair each photo stack with its `run.json`. A Lambda token created from an older preset gets `403 FORBIDDEN` on DishCam files. Create a new token from the current preset and update the Lambda’s `DATA_HUB_API_KEY`; the [Lambda developer docs](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/lambda.md#supported-instrument-types) describe the swap.\n\n\nYou cannot change a token’s scopes after creation. To change them, revoke the token and create a new one.\n\n### Via the API [#via-the-api]\n\nCreating and deleting tokens is admin-only and needs a signed-in browser session (cookie). Tokens cannot manage other tokens.\n\n```sh\ncurl -X POST https://datahub.example.com/api/v1/tokens \\\n -H \"Cookie: session_cookie_here\" \\\n -H \"Content-Type: application/json\" \\\n -d '{\n \"name\": \"FPLC watcher\",\n \"scopes\": [\"instruments:read\", \"instruments:write\", \"watchers:read\", \"watchers:report\", \"runs:create\", \"runs:update\", \"runs:upload\", \"files:update\"],\n \"expires_at\": \"2027-01-01T00:00:00Z\"\n }'\n```\n\n`scopes` is required and must be a non-empty list of explicit scopes. The wildcard `*` and the older coarse scopes `runs:write`, `files:write`, and `watchers:write` are rejected. `expires_at` is optional; if omitted, the token never expires. The request is rejected if it includes `user_id`, because tokens don’t belong to a person. The response includes the plaintext once in the `token` field.\n\n## Use a token [#use-a-token]\n\n### With the watcher [#with-the-watcher]\n\nDuring `data-hub-watcher init`, paste the token when prompted. Or set it first:\n\n```sh\nexport DATA_HUB_API_KEY=dhub_your_token_here\ndata-hub-watcher init\n```\n\n### With the REST API [#with-the-rest-api]\n\n```sh\ncurl https://datahub.example.com/api/v1/instruments \\\n -H \"Authorization: Bearer dhub_your_token_here\"\n```\n\nMore in the [API overview](/docs/api).\n\nTokens don’t authenticate MCP clients, which sign in with OAuth instead. See [MCP overview](/docs/mcp).\n\n## View tokens [#view-tokens]\n\n**Settings → Access Tokens** lists every active token with these columns:\n\n* **Name**\n* **Created by**: the admin who made the token. For tokens created before October 6, 2026, this shows the person the token used to act as.\n* **Token**: the prefix only, because the full token is never stored\n* **Scopes**\n* **Last used**\n* **Expires**\n* **Created**\n\n## Revoke a token [#revoke-a-token]\n\n\n \n Go to **Settings → Access Tokens**.\n \n\n \n Click the delete icon on the token’s row and confirm.\n \n\n\nThe token stops working immediately. Clients using it start receiving `401 Unauthorized`.\n\n```sh\ncurl -X DELETE https://datahub.example.com/api/v1/tokens/ \\\n -H \"Cookie: session_cookie_here\"\n```\n\nA revoked token leaves the list, but Data Hub keeps its record. Wherever the token acted, its name stays visible and is marked **(revoked)**.\n\n## After revoking a token [#after-revoking-a-token]\n\nIf a running watcher used that token:\n\n\n \n Create a new token with the **Watcher** preset.\n \n\n \n On the instrument PC, re-run `data-hub-watcher init` and enter the new token. The new token takes over the existing watcher registration, so you don’t need to deregister the old one first.\n \n\n \n Restart the watcher (`data-hub-watcher watch` or the Windows service).\n \n\n\n## When someone leaves [#when-someone-leaves]\n\nTokens keep working after the admin who created them leaves or their account is deleted, because tokens act as themselves. When someone leaves, check the **Created by** column and revoke any of their tokens that nobody else should keep using.\n\n## Token hygiene [#token-hygiene]\n\n* Name tokens so you can tell which PC or client each belongs to\n* Set expirations for temporary setups\n* Scope to least privilege; see [Token scopes](/docs/security#token-scopes)\n* Revoke immediately when a watcher is decommissioned or a token may have been exposed\n\nTokens are hashed with SHA-256 before storage; plaintext is never persisted.\n" + }, + { + "page": "manage-members", + "title": "Manage members", + "description": "Promote and demote workspace admins from Settings → Members.", + "markdown": "# Manage members (/docs/manage-members)\n\n\n\n\n\n**For:** Administrators\n\nWorkspace members sign in with Google OAuth. Admins decide who else has the admin role. Member vs admin capabilities are summarized in [Security and permissions](/docs/security#roles).\n\n## Open the members list [#open-the-members-list]\n\nGo to **Settings → Members**. Non-admins who open the URL see an “Admins only” message; the settings sidebar hides the entry for them.\n\nThe table lists every signed-in user: name, email, role badge, and an **Admin** toggle.\n\n\"Settings\n\n## Promote or demote [#promote-or-demote]\n\nToggle **Admin** on a row to grant or revoke admin access. Every privileged action re-checks the database, so the change applies to that user’s next one. Their view of the dashboard can lag by up to 5 minutes, because admin-only buttons and sidebar entries come from their [cached session](/docs/security#session-lifetime).\n\nYou cannot demote yourself. That guard prevents locking the workspace out of its last admin. Ask another admin to demote you if needed.\n\n## First admin bootstrap [#first-admin-bootstrap]\n\nThe first admins come from the `ADMIN_EMAILS` environment variable (comma-separated, case-insensitive). Listed users are promoted on every sign-in. After that, any admin can promote others from this page.\n\nThere is no invite flow in the product: people appear here after they sign in successfully to your deployment. Every Google account your OAuth client accepts can sign in, so that client, not this page, controls who appears here. See [Who can sign in](/docs/security#who-can-sign-in).\n" + }, + { + "page": "integrations", + "title": "Set up integrations", + "description": "Connect Data Hub to Linear for feedback reports, and to Slack for personal direct messages and a shared new-run channel, from Settings → Integrations.", + "markdown": "# Set up integrations (/docs/integrations)\n\n\n\n**For:** Administrators and engineers\n\n**Settings → Integrations** holds the connections the whole workspace shares. Only admins see it. The page has three parts:\n\n* **Linear**: where feedback reports go, and where their status comes from\n* **Slack app**: the credentials behind personal Slack direct messages and **Connect to Slack**\n* **Slack channel**: a webhook that posts a message to a shared channel whenever a new run arrives\n\nEach person’s own notification choices are separate, under **Settings → Notifications**. See [Configure notifications](/docs/configure-notifications).\n\n## How secrets are stored [#integration-secrets-key]\n\nData Hub encrypts the secrets you paste on this page before saving them: the Linear client secret and signing secret, and the Slack bot token and client secret. It never shows a saved secret again. To change one, paste a new value and save.\n\nEncryption needs the `INTEGRATION_SECRETS_KEY` environment variable on the web app. It is 64 hexadecimal characters; generate one with `openssl rand -hex 32`, set it in Vercel, and redeploy. Until it is set, saving a secret fails, and the Slack app card says so.\n\nIf the key changes after secrets were saved, Data Hub can’t read them anymore. The affected field says **Can’t read saved value**. Put the old key back, or paste the value again and save.\n\n## Linear for feedback reports [#linear]\n\nFeedback that people send from Data Hub becomes an issue in your Linear workspace, and the report’s status follows that issue. Until an admin saves a Linear team here, the **Feedback** item stays out of the account menu, and sending feedback through the API or an AI assistant returns an error saying feedback isn’t set up. What reporters see is in [Send feedback](/docs/send-feedback).\n\nThe Linear card walks you through four steps.\n\n### 1. Connect your Linear app [#linear-connect]\n\nData Hub signs in to Linear as an app, so you create one app in Linear for it.\n\n\n \n In Linear, go to **Settings → API → OAuth applications** and create a new application.\n \n\n \n Name it **Data Hub**. Linear shows this name on every issue it files.\n \n\n \n Linear asks for a callback URL. Data Hub doesn’t use one, so paste the URL shown in the step, which is your deployment’s address.\n \n\n \n Turn on **Client credentials**, then save.\n \n\n \n Copy the client ID and client secret into the Linear card in Data Hub, and choose **Connect**.\n \n\n\nData Hub checks the credentials with Linear before it saves them. If client credentials are turned off for the app, the card says so; turn them on in the app’s settings in Linear and choose **Connect** again.\n\nIf the credentials belong to a different Linear workspace than the one already connected, Data Hub asks before switching. Switching clears the team, project, labels, and signing secret. Issues already in Linear stay there.\n\n### 2. Choose where reports go [#linear-destination]\n\n\n \n Pick the **Team** that receives reports. Only public teams are listed, because Linear doesn’t show private teams to apps.\n \n\n \n Optionally pick a **Project**.\n \n\n \n Optionally pick a label for each kind of report: **Bug**, **Feature request**, and **Other**. Labels can come from the team or from the whole workspace. Linear adds the matching label to each new issue.\n \n\n \n Choose **Save and turn on feedback**. Everyone sees **Feedback** in their account menu from then on.\n \n\n\nChanging the team later clears the project and labels. Status updates also stop until the Linear webhook from step 3 points at the new team.\n\n### 3. Get status updates from Linear [#linear-status-updates]\n\nA webhook tells Data Hub when an issue changes, so it can update the report and tell the person who sent it when the issue closes.\n\n\n \n In Linear, go to **Settings → API** and choose **New webhook**.\n \n\n \n Paste the **Webhook URL** shown in the step. It ends in `/api/v1/integrations/linear/webhook`.\n \n\n \n For **Team**, choose the team from step 2. Under events, turn on **Issues** only.\n \n\n \n Save the webhook, then copy its signing secret into **Signing secret** in Data Hub and save.\n \n\n\nData Hub uses the signing secret to check that each update really comes from Linear. The step shows **Waiting for an update from Linear** until the first one arrives. Any change to an issue in the team sends one, including the test report in the next step.\n\n### 4. Send a test report [#linear-test-report]\n\nChoose **Send test report**. Data Hub files a report as you and links to the new Linear issue. Move that issue to **Done** in Linear. The step confirms that the issue closed and that you got a Data Hub notification. You can also skip the test.\n\n### After setup [#linear-after-setup]\n\nOnce setup is finished, the Linear card shows a summary: the connected workspace, where reports go, whether status updates are working and when the last one arrived, and what reporters see for each Linear status.\n\n| When the Linear issue moves to | The reporter sees |\n| --------------------------------------- | ----------------- |\n| **Done**, or any other completed status | Resolved |\n| **Canceled** | Declined |\n| **Duplicate** | Declined |\n| Any other status | Open |\n\nData Hub notifies the reporter once, when the issue first closes. Moving an issue from one closed status to another, such as **Done** to **Released**, doesn’t notify again. Reopening an issue and closing it again does.\n\nIf Data Hub keeps rejecting updates from Linear, **Status updates** shows **Not working**. The usual cause is a wrong signing secret, or Linear turning the webhook off after repeated failures. Paste the signing secret again from the webhook’s page in Linear, and check that the webhook is still turned on.\n\n**Disconnect Linear** turns feedback off. Issues already in Linear stay there.\n\n## Slack app for direct messages [#slack-app]\n\nPersonal Slack direct messages and the **Connect to Slack** button use one shared Slack app. Create the app once in the [Slack API portal](https://api.slack.com/apps), then give Data Hub its credentials. You can reuse an existing Slack app, including the one behind the channel webhook.\n\n### Create the app [#create-the-slack-app]\n\n\n \n In the Slack API portal, choose **Create New App → From scratch**, or open an existing app.\n \n\n \n Under **OAuth & Permissions**, add these scopes:\n\n **Bot Token Scopes:**\n\n | Scope | Purpose |\n | ------------ | -------------------------------------------- |\n | `chat:write` | Send direct messages with `chat.postMessage` |\n\n **User Token Scopes:**\n\n | Scope | Purpose |\n | --------- | ----------------------------------- |\n | `openid` | Sign in with Slack (OpenID Connect) |\n | `profile` | Display name in the identity token |\n \n\n \n Under **Redirect URLs**, add one callback URL per environment. The path is always `/api/v1/settings/slack/callback`:\n\n | Environment | Example URL |\n | ----------- | -------------------------------------------------------------------- |\n | Local | `http://localhost:3000/api/v1/settings/slack/callback` |\n | Staging | `https://staging.datahub.example.com/api/v1/settings/slack/callback` |\n | Production | `https://datahub.example.com/api/v1/settings/slack/callback` |\n\n `SLACK_REDIRECT_URI`, or the request origin plus that path when unset, must match a listed URL exactly.\n \n\n \n Under **Install App**, choose **Install to Workspace** (or **Reinstall** after changing scopes).\n \n\n\n### Give Data Hub the credentials [#slack-app-credentials]\n\nEnter the credentials in the **Slack app** card under **Settings → Integrations**, or set them as environment variables on the web app. A value saved in the card takes effect right away, with no redeploy, and is used instead of the matching environment variable.\n\n| Card field | Environment variable | Where to find it | Required |\n| ------------------------ | --------------------- | ------------------------------------------------------- | --------------------------------------------------------- |\n| **Bot token** | `SLACK_BOT_TOKEN` | **Install App** → Bot User OAuth Token (`xoxb-…`) | Yes |\n| **Client ID** | `SLACK_CLIENT_ID` | **Basic Information** → App Credentials | Yes |\n| **Client secret** | `SLACK_CLIENT_SECRET` | **Basic Information** → App Credentials | Yes |\n| **Allowed workspace ID** | `SLACK_TEAM_ID` | Workspace URL segment `T…` in `app.slack.com/client/T…` | No, but recommended so connections stay on your workspace |\n\nEach field shows where its value comes from: **Saved here**, **Using** followed by the variable name, or **Not set**. **Remove saved value** makes the environment variable the source again. If Data Hub can’t read a saved value, for example because `INTEGRATION_SECRETS_KEY` changed, it uses the environment variable in the meantime.\n\nTwo settings exist only as environment variables, and changing them needs a redeploy:\n\n| Variable | Purpose | Required |\n| -------------------- | ------------------------------------------------------------------------------------------------------------------ | -------------------------------- |\n| `SLACK_STATE_SECRET` | Signs the short-lived value Data Hub checks while a person connects Slack. Generate with `openssl rand -base64 32` | No (falls back to `AUTH_SECRET`) |\n| `SLACK_REDIRECT_URI` | Full callback URL, when it differs from the request origin | No |\n\nMembers can then connect under **Settings → Notifications**. Deploy runbooks are linked from [Deploy Data Hub](/docs/self-hosting).\n\n## Slack channel for new runs [#slack-channel]\n\nThe **Slack channel** card holds a Slack incoming webhook. Data Hub posts one message to that channel each time a new run is reported. This is separate from personal direct messages, which each person turns on for themselves.\n\n\n \n In your Slack app, open **Incoming Webhooks**, turn them on, and add a webhook for the channel you want.\n \n\n \n Copy the webhook URL. It starts with `https://hooks.slack.com/services/`.\n \n\n \n Paste it into the **Slack channel** card and save. The card shows **Configured** and who last changed it.\n \n\n\nTo change channels, paste a new webhook URL. To stop the messages, remove the webhook. This setting has no environment variable.\n" + }, + { + "page": "api", + "title": "Overview", + "description": "Authenticate to the Data Hub REST API, make a first request, and understand pagination, filtering, token attribution, and errors.", + "markdown": "# Overview (/docs/api)\n\n\n\n**For:** Developers and integrators\n\nThe Data Hub API is served by the web application at `/api/v1/`. The watcher, Lambda, and dashboard all call it. Data Hub is self-hosted, so the base URL is your deployment host plus `/api/v1`. Examples use `https://datahub.example.com`; substitute your host.\n\nPer-endpoint request and response shapes live in the generated pages under this section (same sidebar). Each deployment also publishes OpenAPI 3.1 at `/api/v1/openapi.json`.\n\n## Authentication [#authentication]\n\nTwo methods (full model in [Security and permissions](/docs/security)):\n\n* **Session cookies**: web dashboard (Google OAuth). Session callers implicitly hold every scope.\n* **Bearer tokens**: `Authorization: Bearer dhub_…` for the watcher, the Lambda, and scripts\n\n```sh\ncurl https://datahub.example.com/api/v1/instruments \\\n -H \"Authorization: Bearer dhub_your_token_here\"\n```\n\nEvery token request needs a [scope](/docs/security#token-scopes). A missing scope returns `403 FORBIDDEN`. Some mutations also require the admin role; see [Admin-gated operations](/docs/security#admin-gated-operations).\n\nSome routes accept only one of the two methods:\n\n* The routes the watcher and the Lambda call accept only tokens, and return `401 UNAUTHORIZED` to a session. These cover watcher registration and heartbeats, upload links, file updates, run creation and updates, and instrument creation.\n* Claiming runs, sending feedback, reviewing feedback, and managing tokens need a signed-in person, so tokens can’t call them.\n\nMint tokens in the UI: [Issue and revoke tokens](/docs/manage-tokens).\n\n## First request [#first-request]\n\nList instruments:\n\n```sh\ncurl https://datahub.example.com/api/v1/instruments \\\n -H \"Authorization: Bearer dhub_your_token_here\"\n```\n\nFetch the OpenAPI document (no auth required):\n\n```sh\ncurl https://datahub.example.com/api/v1/openapi.json\n```\n\nThe document covers the integrator surface: instruments, runs, comments, files, watchers, archive jobs, search, and `POST /notifications/dispatch`. Routes that only the dashboard uses are left out: `/tokens`, `/users`, `/feedback`, `/settings/*`, and the notification inbox. The MCP transport is documented separately.\n\n## Actions taken by a token [#actions-taken-by-a-token]\n\nA token acts under its own name, not as a person, so responses name the token when it did something:\n\n* A comment posted by a token has `user: null` and a `token` object with the token’s `id` and `name`. A comment posted by a person has `token: null`.\n* A run deleted by a token has `deleted_by_token` set, alongside `deleted_at`.\n\nRead both fields when you display who did something. See [How a token acts](/docs/manage-tokens#how-a-token-acts).\n\n## Send notifications [#send-notifications]\n\n`POST /notifications/dispatch` sends a short message to chosen people, with an optional link to a run. It needs a token with the `notifications:create` scope. Each person’s **Service notifications** setting decides whether the message appears in the notification bell, as a Slack direct message, or both. The message shows the token’s name as its sender. Request fields are on the generated [dispatch notifications](/docs/api/notifications/dispatchNotifications) page.\n\n## Pagination and filtering [#pagination-and-filtering]\n\nList endpoints that return pages (notably run search) accept:\n\n* `page`: 1-based page index\n* `per_page`: page size, 1 to 100\n\nResponses include pagination metadata such as `page`, `per_page`, `total`, and `total_pages`.\n\nRun report items page by index instead, with `offset` and `limit` (max 200). Their responses return `offset`, `limit`, and `total`.\n\nRun search also supports filters (instrument, date range, attribution via `ranBy`, and instrument-specific metadata). Exact query parameters are on each generated operation page in the sidebar under this **API** section.\n\n## Error responses [#error-responses]\n\nErrors use a nested envelope:\n\n```json\n{\n \"error\": {\n \"code\": \"NOT_FOUND\",\n \"message\": \"Instrument not found.\",\n \"details\": null\n }\n}\n```\n\nCommon codes: `VALIDATION_ERROR` (400), `UNAUTHORIZED` (401), `FORBIDDEN` (403), `NOT_FOUND` (404), `CONFLICT` (409). Scope failures use `code: \"FORBIDDEN\"` with a message naming the missing scope.\n\nFeedback routes have two more. `LINEAR_UNAVAILABLE` (502) means Data Hub couldn’t reach Linear, where reports are stored. `FEEDBACK_NOT_CONFIGURED` (503) means no admin has connected Linear yet; see [Set up integrations](/docs/integrations#linear).\n\n## MCP [#mcp]\n\n`GET` and `POST` `/mcp/v1` expose the MCP server over Streamable HTTP, outside the `/api/v1` tree. Clients authenticate with OAuth rather than a personal access token, and hold `read` to connect plus `write` to call any tool that changes data. See [MCP overview](/docs/mcp).\n" + }, + { + "page": "mcp", + "title": "Overview", + "description": "Connect an AI client to Data Hub over the Model Context Protocol, including OAuth sign-in, the read and write scopes, and per-client setup.", + "markdown": "# Overview (/docs/mcp)\n\n\n\n**For:** Developers and integrators\n\nData Hub exposes a Model Context Protocol (MCP) server at `/mcp/v1` so AI clients can query instruments, runs, files, and watcher health. Clients authenticate by sending you through a browser sign-in.\n\nCatalogs for [tools](/docs/mcp/tools), [prompts](/docs/mcp/prompts), and [resources](/docs/mcp/resources) are generated from the live server schema, so they match what your client discovers when it connects.\n\n## Authentication [#authentication]\n\nThe MCP server authenticates with OAuth. All you need to provide is the endpoint URL; for example, `https://datahub.example.com/mcp/v1`.\n\nThe first connection opens your browser:\n\n\n \n Sign in to Data Hub with Google, unless you already have a session open.\n \n\n \n Read the **Authorize application** screen. It names the client, the host it redirects to, and the access it’s asking for.\n \n\n \n Click **Allow**.\n \n\n\nYour client stores the access token it receives and renews it without asking again. **Allow** grants everything on the screen: there are no per-permission checkboxes, so click **Deny** if any line looks wrong.\n\n### Scopes [#scopes]\n\nMCP uses two scopes rather than the `:` scopes that gate REST requests:\n\n* **`read`**: required to connect at all, and covers every read-only tool, resource, and prompt\n* **`write`**: additionally required by tools that change something, including claiming runs, commenting, requesting uploads, reprocessing, and deleting\n\nMost clients ask for both, because the server advertises both. A client granted only `read` still connects and still reads; its first mutating tool call fails with a message naming the missing scope.\n\nEach entry in the [tools catalog](/docs/mcp/tools) lists the scope its REST equivalent requires, which tells you what data the tool touches. Over MCP, only `read` and `write` are enforced.\n\nThe client acts as you, with your role. Admin-only actions, such as deleting someone else’s comment, work over MCP only when you are an admin. Claiming runs and sending feedback work because a signed-in person is behind the client, even though API tokens can’t do either.\n\n## Connect a client [#connect-a-client]\n\nExamples use `https://datahub.example.com`; substitute your deployment host.\n\n### Claude Code [#claude-code]\n\nRegister the server with [`claude mcp add`](https://code.claude.com/docs/en/mcp-quickstart). Run this in your terminal, not inside a `claude` session:\n\n```bash\nclaude mcp add --transport http --scope user data-hub https://datahub.example.com/mcp/v1\n```\n\nConfirm that the server was added by running `claude mcp list`. Then, start a session with `claude` and run `/mcp` to authenticate.\n\n### Claude Desktop [#claude-desktop]\n\nRemote servers go through **Settings → Connectors → Add custom connector**. The `claude_desktop_config.json` file handles local servers only. Paste `https://datahub.example.com/mcp/v1` as the URL, click **Add**, then click **Connect** to sign in. Leave the OAuth client fields under **Advanced settings** empty, since Data Hub registers clients on demand.\n\n\n Claude reaches your server from Anthropic’s infrastructure rather than from your own machine, so the deployment has to be reachable from the public internet. A deployment running on `localhost` can’t be added this way; use Claude Code or Cursor for local work.\n\n\n### Cursor [#cursor]\n\nAdd the server to `.cursor/mcp.json` in your project, or to `~/.cursor/mcp.json` for every project:\n\n```json\n{\n \"mcpServers\": {\n \"data-hub\": {\n \"url\": \"https://datahub.example.com/mcp/v1\"\n }\n }\n}\n```\n\nOpen **Settings → Tools & MCP**. The server appears with a **Needs authentication** label; click it to start the browser flow. The tools show up once you approve the grant.\n\n### Other clients [#other-clients]\n\nAny client that speaks Streamable HTTP and the MCP authorization spec needs one setting, the URL `https://datahub.example.com/mcp/v1`. For local development, point at `http://localhost:3000/mcp/v1` instead.\n\nClients that speak MCP but not OAuth can bridge through [`mcp-remote`](https://www.npmjs.com/package/mcp-remote), which runs the browser flow on their behalf.\n\n## Questions about Data Hub [#questions-about-data-hub]\n\nThe assistant can answer questions about Data Hub itself, such as how to install a watcher or what a run status means. It searches the documentation and the [changelog](/docs/send-feedback#changelog), reads the page that fits, and links it in its answer. Two tools do this, and both need only the `read` scope:\n\n* `search_docs` returns up to five matching sections, each with a short excerpt and a link\n* `read_doc` returns one page, or one section of a page, as Markdown\n\nThe answers come from the docs that ship with your deployment, so they match the version you run, including a staging deployment that is ahead of the published site. Clients that let you attach resources by hand can also attach any page as `datahub://docs/`, for example `datahub://docs/manage-tokens`.\n\nThe assistant reads the guides and the changelog. It doesn’t read the generated [API](/docs/api) endpoint pages or the [tools](/docs/mcp/tools), [prompts](/docs/mcp/prompts), and [resources](/docs/mcp/resources) catalogs, because the client already receives the catalog when it connects. When the docs don’t cover a question, the assistant says so and offers to [send feedback](#feedback) about the gap.\n\n## Interactive run reports [#interactive-run-reports]\n\nClients that support MCP Apps, an extension of the Model Context Protocol, show the same interactive run report as the web app when the assistant calls `get_run_report`. The report includes image carousels, plate maps, and the Aunty plate with its well charts. Clients without MCP Apps support get the same data as text and describe it instead.\n\n## Feedback [#feedback]\n\nThe assistant can send [feedback about Data Hub](/docs/send-feedback) for you, for example after a tool call fails. It shows you the draft first and sends it only after you agree. Sending feedback needs only the `read` scope. The assistant can also list the reports you sent and read one in full, by report ID or by Linear issue ID such as `ENG-1476`. Feedback tools return an error saying feedback isn’t set up until an admin [connects Linear](/docs/integrations#linear).\n\nProblems with one run’s data belong in a run comment instead, which the assistant can also post.\n\n## Machine-readable catalog [#machine-readable-catalog]\n\nFetch the same catalog these pages render from:\n\n```bash\ncurl https://datahub.example.com/mcp/v1/schema.json\n```\n\nIt lists every tool, prompt, and resource, with JSON Schema for arguments and each tool's response. The schema document needs no authentication.\n" + }, + { + "page": "concepts", + "title": "Concepts", + "description": "Short definitions of the Data Hub vocabulary: instruments, watchers, runs, statuses, processing, and access.", + "markdown": "# Concepts (/docs/concepts)\n\n\n\n**For:** Everyone\n\nShort definitions of the vocabulary the rest of these docs use. If you’ve arrived mid-page and a term is unfamiliar, it’s defined here. Terms are grouped by what they describe: the equipment, the data, the rules that turn files into runs, and who can do what.\n\n## Equipment and fleet [#equipment-and-fleet]\n\n### Instrument [#instrument]\n\nA piece of lab equipment registered in Data Hub, identified by a kebab-case **instrument ID** (e.g., `akta-fplc`, `bio-rad-cfx96`). The ID is permanent, because it becomes the storage key prefix and the identifier used across the system. A human-readable display name sits alongside it.\n\nAn instrument has three states. The dashboard groups instruments into a tab per state and shows connection health separately:\n\n| State | Shown as | Meaning |\n| ---------- | --------------------------------------------- | ---------------------------------------------------------------------------- |\n| `pending` | **Pending** tab | Registered but not yet confirmed by an admin. A watcher can’t upload for it. |\n| `active` | **Active** tab, with Online or Offline health | Confirmed and accepting uploads. |\n| `inactive` | **Retired** tab | Deactivated. No longer accepting uploads, but its runs stay browsable. |\n\nSee [Set up an instrument](/docs/set-up-an-instrument), [Manage instruments](/docs/manage-instruments), and [Retire an instrument](/docs/retire-an-instrument).\n\n### Instrument type [#instrument-type]\n\nSeparate from the instrument ID. The type tells Data Hub which [preprocessor](#preprocessor) to run on files from this instrument, so it controls what metadata, badges, and reports you get.\n\nAdmins set it from the instrument page: open the three-dot menu, choose **Edit details**, and pick a supported type. An instrument with no matching type still stores and serves raw files. It gets no extracted metadata and no processed artifacts. See [Instrument data preprocessing](/docs/instrument-preprocessing).\n\n### Watcher [#watcher]\n\nThe `data-hub-watcher` command-line program that runs on an instrument PC. It monitors a directory, groups new files into runs, uploads them to cloud storage, and reports status to the API. Each instrument can have at most one active watcher at a time.\n\nA watcher is registered with the API and carries one registration ID per [environment](#environment). Deregistering a watcher is a [soft delete](#soft-delete): its history stays visible for auditing. See [Manage the watcher fleet](/docs/manage-watchers).\n\n### Watcher status [#watcher-status]\n\nWhat the **Watchers** page shows for each watcher, derived from its most recent [heartbeat](#heartbeat):\n\n| Status | Meaning |\n| ---------------- | ------------------------------------------------------------------------------------------------------------- |\n| **Online** | A heartbeat arrived in the last 5 minutes. |\n| **Unresponsive** | No heartbeat for more than 5 minutes. The process crashed, the PC is off, or the network is blocking the API. |\n| **Stopped** | The watcher shut down cleanly and said so in its last heartbeat. |\n| **Registered** | The watcher registered but hasn’t sent its first heartbeat yet. |\n\nAn unresponsive watcher keeps its registration and history, so you can read its last events to work out what happened. See [Troubleshoot a watcher](/docs/troubleshoot-a-watcher).\n\nInstrument pages and the instruments table sum this up per instrument: **Online** if any of its watchers is online, **Offline** if none is, and **No Watcher** if none has registered.\n\n### Heartbeat [#heartbeat]\n\nA status ping the watcher sends to the API every 60 seconds. It carries the watcher version, instrument ID, watch directory, upload mode, activity counters, and uptime, and it drives the **Last Heartbeat** column and the online health shown on the dashboard. The same tick runs the auto-updater. The upload queue is checked separately, also once a minute, so a slow upload can’t delay a heartbeat.\n\n### Events [#events]\n\nA per-watcher log of lifecycle moments and errors: `watcher_started`, `run_reported`, `file_uploaded`, `update_succeeded`, and `error` (with a `details.kind` discriminator), among others. Events are batched and flushed on each heartbeat and are visible on the dashboard, which makes them the primary tool for diagnosing a watcher remotely. See [Troubleshoot a watcher](/docs/troubleshoot-a-watcher).\n\n### Environment [#environment]\n\nWhich Data Hub deployment a watcher talks to: `staging`, `production`, or `preview` (a Vercel preview deployment, which needs an explicit API base URL). Staging and production are separate deployments with separate databases, so a single PC keeps an independent registration and local state per environment and can switch between them.\n\n## Data [#data]\n\n### Run [#run]\n\nA logical grouping of files produced by one acquisition. The watcher assigns each file a **run ID** using its configured [run detection](#run-detection) method, then reports the run to the API. The first file for a run creates it; later files are added incrementally to the same run’s manifest.\n\n### Run status [#run-status]\n\nHow far a run’s files have got from the instrument to the dashboard. These are the values you can filter by under **Add filter** on any run list:\n\n| Status | Meaning |\n| -------------- | ------------------------------------------------------------------------------------------------------------ |\n| Failed | At least one file stopped on an error. [Reprocess](/docs/browse-runs#reprocess-and-delete) after a fix. |\n| Stalled | At least one file has been processing for more than 20 minutes without finishing. Reprocess it to try again. |\n| Pending upload | The watcher found the files but hasn’t sent them yet. |\n| Uploaded | The files are in storage. Processing hasn’t started, or this instrument type has no preprocessor. |\n| Processing | The preprocessor is reading the files, extracting metadata, and building reports. |\n| Completed | Processing finished. |\n| Empty | The run has no files. |\n\nWhen a run’s files are in different states, the run shows the first status in this table that applies.\n\n### File and file category [#file-and-file-category]\n\nEvery file in a run carries a category, shown as a label in the run’s files table:\n\n* **Raw**: what the instrument wrote, uploaded unchanged\n* **Processed**: what a [preprocessor](#preprocessor) generated from a raw file, such as a CSV of well values, a contrast-adjusted PNG, or a preview JPG\n\nBoth categories download the same way, and file filters on the run page work on this label. See [Download files](/docs/browse-runs#download-files).\n\n### Preprocessor [#preprocessor]\n\nThe server-side code that reads a raw file, extracts metadata, and writes processed files. It runs in an AWS Lambda when a file arrives in raw storage, or when you choose **Reprocess**. Which preprocessor runs depends on the [instrument type](#instrument-type), and not every type has one. Per-type inputs and outputs are in [Instrument data preprocessing](/docs/instrument-preprocessing).\n\n### Run archive [#run-archive]\n\nThe zip that **Download all** builds for a run. It covers the run’s active files and respects any file filter you have applied. Large archives build in the background as an archive job, so the first request can return a wait time rather than a file.\n\n### Attribution [#attribution]\n\nThe record of who ran a run. Use **Claim** on the run to attribute yourself. Attribution is self-only: you claim or unclaim your own user, and you can’t attribute someone else. API tokens can’t claim runs. Claims drive **My runs** and the “ran by” filters, and claiming a run subscribes you to its comments. See [Claim who ran it](/docs/browse-runs#claim-who-ran-it).\n\n## Turning files into runs [#turning-files-into-runs]\n\n### Run detection [#run-detection]\n\nThe rule that maps a file to a run ID. Two broad strategies:\n\n* **Prefix**: a regex extracts the run ID from the filename. The default `^([^_]+)` takes everything before the first underscore, so `RUN001_data.csv` → `RUN001`\n* **Directory**: each subdirectory under the watch directory is its own run, so `RUN001/data.csv` → `RUN001`\n\nThe full set of presets, and the custom-regex option, is in the [Configuration reference](/docs/configuration-reference#run-detection).\n\n### Stability period [#stability-period]\n\nHow many seconds a file must stay unchanged (same size and modification time) before the watcher treats it as fully written and eligible for upload. The default is 5 seconds; raise it for instruments that write large files slowly. A file that keeps changing past `max_stability_wait_seconds` (default 300, or 5 minutes) is abandoned with a `stability_timeout` error event.\n\n### Upload mode [#upload-mode]\n\nHow files get from the instrument to cloud storage:\n\n* **auto**: files upload immediately after they’re detected and stabilized. With watcher 1.2.1 and later, the watcher also uploads files someone requests from the run page\n* **manual**: runs are reported to the server, but files aren’t uploaded until approved through the server-side upload queue, which the watcher checks once a minute. Use this when uploads need human approval\n\n### Initial scan [#initial-scan]\n\nWhat the watcher does with files already sitting in the watch directory the first time an environment is entered. `production` uploads the existing backlog (`full`). `staging` and `preview` record it as a baseline and skip it (`new-only`), so test environments aren’t flooded with history. See the [Configuration reference](/docs/configuration-reference#initial-scan).\n\n### Dismiss [#dismiss]\n\nFor instruments in [manual upload mode](#upload-mode), dismissing a detected file tells the watcher not to upload it. Dismissing differs from deleting: the file never reached Data Hub storage, so there’s nothing to restore. See [Manage the watcher fleet](/docs/manage-watchers#approving-manual-uploads).\n\n### Soft delete [#soft-delete]\n\nData Hub marks things deleted rather than erasing them, so history survives for auditing. Deleting a run hides it from run lists without removing its stored files, and **Show deleted runs** brings it back into view. Deregistering a watcher keeps its events and heartbeats. Anyone signed in can restore a deleted run from the run page.\n\n## People and access [#people-and-access]\n\n### Role [#role]\n\nData Hub has two roles, **Member** and **Admin**. Members sign in, browse instruments and runs, claim and comment on runs, send feedback, and read the token list. Admins can also confirm and update instruments, create and revoke tokens, promote or demote other users, delete any comment, review feedback, and manage integrations. Full permission tables are in [Security and permissions](/docs/security#roles).\n\n### Personal access token [#personal-access-token]\n\nA `dhub_`-prefixed bearer token that authenticates the watcher, and other API clients, with the Data Hub API. Tokens carry permission [scopes](#scope) and are created and revoked by admins. The plaintext is shown once, at creation.\n\nA token acts as itself, under its own name. Comments it posts and runs it deletes show the token’s name with a key icon, not a person. Because a token isn’t a person, it can’t claim runs or send feedback. See [Issue and revoke tokens](/docs/manage-tokens).\n\n### Scope [#scope]\n\nA permission string carried by a token, such as `runs:read`. A token-authenticated request is refused with `403 FORBIDDEN` when the token’s scopes don’t cover what the route requires. Dashboard sessions hold every scope, so scope checks only apply to token requests. The scope table is in [Security and permissions](/docs/security#token-scopes).\n\n### Feedback report [#feedback-report]\n\nA bug report or feature request about Data Hub itself, sent from the account menu or by an AI assistant. Each report is stored as an issue in your workspace’s Linear, and its status (Open, Resolved, or Declined) follows that issue. See [Send feedback and see what’s new](/docs/send-feedback).\n" + }, + { + "page": "architecture", + "title": "Architecture", + "description": "How Data Hub fits together: the watcher, S3 storage, the Lambda processor, the API and web app, and the data flow from instrument to dashboard.", + "markdown": "# Architecture (/docs/architecture)\n\n\n\n**For:** Engineers and operators\n\nData Hub has four components that coordinate through S3, a REST API, and a shared\nPython library. This page is the conceptual map; the deployment mechanics live in\nthe [first-time deployment guide](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md)\nin the data-hub developer docs.\n\n## System overview [#system-overview]\n\n|raw files| S3[S3 raw]\n S3 -->|trigger| L[Lambda]\n W -->|heartbeats, runs| API[API]\n L -->|processed data| Sproc[S3 processed]\n L -->|results| API\n L -->|run-zip multipart| Sarc[S3 archives]\n API -->|presigned URLs| S3\n API -->|presigned URLs| Sarc\n API -->|build archive| L\n API --> DB[(PostgreSQL)]\n API --> UI[Web app]\n MCP[MCP clients] -->|tools, resources| API\"\n/>\n\n## Components [#components]\n\n| Component | Package | Description |\n| -------------- | ------------------ | -------------------------------------------------------------------------------------------- |\n| Web app | `data-hub-web` | Next.js web application, REST API, and MCP server. Deployed on Vercel. |\n| Lambda | `data-hub-lambda` | AWS Lambda function triggered by S3 uploads. Runs instrument-specific processing. |\n| Watcher | `data-hub-watcher` | CLI on lab instrument PCs. Detects new files, uploads them to S3, reports status to the API. |\n| Shared library | `data-hub-shared` | Shared Python library: S3 utilities, instrument enums, test infrastructure. |\n\n## Data flow [#data-flow]\n\n### Automatic upload (auto mode) [#automatic-upload-auto-mode]\n\n1. A lab instrument writes output files to a watched directory.\n2. The **watcher** detects new stable files via filesystem events.\n3. It groups files into runs and reports each run to the **API**.\n4. It uploads raw files to **S3** at the key `{instrument_id}/{run_id}/{filename}`.\n5. The S3 upload triggers the **Lambda**.\n6. Lambda dispatches to the instrument’s processor, which reads the file for preprocessing. Very large files can be read from a read-only mount of the raw bucket instead of being downloaded, when the deployment turns that option on.\n7. Lambda creates/updates the run and files via the **API**, which sends a **Slack** notification once per new run.\n8. Users view the run in the **web dashboard**.\n\n### Manual upload (manual mode) [#manual-upload-manual-mode]\n\nSteps 1–3 are the same, but the watcher doesn’t upload immediately:\n\n4. The server adds files to an upload queue.\n5. Once a minute the watcher checks the queue and uploads the files someone requested.\n6. Steps 5–8 from auto mode follow.\n\nFrom watcher 1.2.1, auto-mode watchers check the same queue, so **Upload** on a run page works for them too.\n\n## Key design decisions [#key-design-decisions]\n\n* **S3 is the integration boundary.** The watcher and Lambda never talk directly; S3 is the durable hand-off: the watcher writes, Lambda reads.\n* **API-driven coordination.** The watcher registers, syncs its YAML config, and heartbeats, which is what powers dashboard health and the upload queue.\n* **Pre-signed URLs from the API.** The web app mints pre-signed S3 upload/download URLs so bytes move directly to/from S3 without routing through the API. On Vercel the app assumes an AWS Identity and Access Management (IAM) role via OpenID Connect (OIDC), with no long-lived AWS keys.\n* **Lambda-built run archives.** “Download all” delegates to the Lambda, which zips files into a separate archives bucket via S3 multipart upload; the web app then 302s the browser to a short-lived presigned URL. Bytes never traverse Vercel.\n* **Shared library for contracts.** Instrument IDs, S3 utilities, and environment config live in `data-hub-shared` so Lambda and the watcher stay consistent.\n* **MCP for AI access.** The web app serves an MCP endpoint at `/mcp/v1` exposing tools, resources, and prompts to AI clients. Clients authenticate with OAuth against the same Better Auth instance that signs people in to the dashboard, so an agent acts as the person who approved it. See [MCP overview](/docs/mcp).\n\n## Branch and environment strategy [#branch-and-environment-strategy]\n\n| Branch | Purpose |\n| ------------ | ----------------------------------------------- |\n| `staging` | Pre-production. Pull requests merge here first. |\n| `production` | Live. Changes are promoted from `staging`. |\n\nFeature branches target `staging` via pull requests (PRs). Continuous integration\n(CI) runs on every PR and on merges to both branches; merges to `staging` and\n`production` deploy each component to its environment automatically. Staging and\nproduction are independent deployments with separate databases.\n" + }, + { + "page": "configuration-reference", + "title": "Watcher configuration", + "description": "Every config.yaml field, the ~/.data-hub file layout, environment variables, run-detection presets, upload modes, and the initial-scan behavior.", + "markdown": "# Watcher configuration (/docs/configuration-reference)\n\n\n\n**For:** Lab operators\n\nThe complete reference for the watcher’s configuration: the `config.yaml` schema,\nwhere files live, the environment variables that influence it, and the behaviors\n(run detection, upload modes, initial scan) those fields drive.\n\n## File locations [#file-locations]\n\nEverything the watcher persists lives under `~/.data-hub/`:\n\n| Path | Contents |\n| -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `~/.data-hub/config.yaml` | The main config (below). Override with `--config` or `DATA_HUB_CONFIG_PATH`. |\n| `~/.data-hub/.env.` | The API key for an environment (e.g. `.env.staging`, `.env.production`, `.env.preview`). The legacy `~/.data-hub/.env` is still loaded, with the per-environment file taking precedence. |\n| `~/.data-hub/watcher-.db` | Per-environment SQLite state (uploaded files, runs, detected files, baseline). |\n| `~/.data-hub/watcher.log` | Rotating log (10 MB × 5 backups). On Windows: `C:\\ProgramData\\DataHubWatcher\\watcher.log`. |\n| `~/.data-hub/upgrade-worker.log` | Windows uv-tool upgrade transcript. |\n\n## Environment variables [#environment-variables]\n\n| Variable | Purpose |\n| ---------------------------- | --------------------------------------------------------------------------------------------- |\n| `DATA_HUB_API_KEY` | Supplies the API key, skipping the `init` prompt. Saved into the per-environment `.env` file. |\n| `DATA_HUB_CONFIG_PATH` | Override the config file path (same as `--config`). |\n| `DATA_HUB_WATCHER_LOG_LEVEL` | Set to `DEBUG` (in the service’s `.env` file) to enable debug logging without a reinstall. |\n\n## `config.yaml` [#configyaml]\n\n```yaml\nversion: 1\nenvironment: production # \"staging\", \"production\", or \"preview\"\napi_base_url: null # required when environment is \"preview\"\nwatcher_ids: # one registration id per environment\n production: \ninitial_scan: null # null (default), \"full\", or \"new-only\"\ninstrument:\n id: akta-fplc # kebab-case instrument ID\n watch_directory: /path/to/data\n file_patterns:\n - \"*.csv\"\n - \"*.xlsx\"\n enabled: true\n upload_mode: auto # \"auto\" or \"manual\"\n stability_period_seconds: 5 # 1–300\n max_stability_wait_seconds: 300 # 1–86400; must be >= stability_period_seconds\n run_detection:\n pattern: '^([^/]+)/' # regex with one capture group (run ID)\n recursive: true\n```\n\n### Field reference [#field-reference]\n\n| Field | Type | Description |\n| --------------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| `version` | int | Config schema version. Currently `1`. |\n| `environment` | enum | `staging`, `production`, or `preview`. |\n| `api_base_url` | string \\| null | Required only for `preview` (the deployment’s `/api/v1` URL). |\n| `watcher_ids` | map | One watcher registration id per environment, assigned by the API. |\n| `initial_scan` | enum \\| null | `null` (env default), `full`, or `new-only`. See [Initial scan](#initial-scan). |\n| `instrument.id` | string | Kebab-case instrument ID. Permanent; the storage key prefix. |\n| `instrument.watch_directory` | string | Absolute path the instrument writes to. |\n| `instrument.file_patterns` | string\\[] | Glob patterns; only matching files are uploaded. |\n| `instrument.enabled` | bool | Whether the watcher processes this instrument. |\n| `instrument.upload_mode` | enum | `auto` or `manual`. See [Upload modes](#upload-modes). |\n| `instrument.stability_period_seconds` | int (1–300) | Seconds a file must stay unchanged before it’s considered written. Default `5`. |\n| `instrument.max_stability_wait_seconds` | int (1–86400) | Seconds a still-changing file may stay pending before the watcher abandons it with a `stability_timeout` event. Must be ≥ `stability_period_seconds`. Default `300` (5 minutes). |\n| `instrument.run_detection.pattern` | regex | One-capture-group regex applied to each file’s path relative to `watch_directory`. |\n| `instrument.run_detection.recursive` | bool | `true` watches subdirectories; `false` only the top level. |\n\n\n A config written by an older watcher used a single top-level `watcher_id`. It’s\n migrated transparently on load: lifted into `watcher_ids` under the active\n environment and dropped on the next save.\n\n\n## Run detection [#run-detection]\n\nThe `pattern` regex is applied with `re.search` to each file’s path relative to\n`watch_directory`, with backslashes normalized to `/`. Capture group 1 is the run\nID, and the pattern must have exactly one capture group. In YAML, single-quote\npatterns so backslash sequences like `\\d` don’t need escaping.\n\nThe `init` wizard offers these presets (or supply a custom regex):\n\n| Preset | Pattern | Recursive | Run ID is… |\n| ---------------------- | ------------------------------ | --------- | ----------------------------------------------------------------- |\n| Filename prefix | `^([^_]+)` | no | everything before the first `_`: `RUN001_data.csv` → `RUN001` |\n| Top subdirectory | `^([^/]+)/` | yes | the top-level folder: `RUN001/data.csv` → `RUN001` |\n| Deepest subdirectory | `([^/]+)/[^/]+$` | yes | the immediate parent folder: `plate-a/well-b/data.csv` → `well-b` |\n| Timestamp subdirectory | `(?:^\\|/)(\\d{8}_\\d{6}_\\d{3})/` | yes | a `YYYYMMDD_HHMMSS_fff` folder anywhere in the path |\n| Filename stem | `^(?:.+/)?([^/]+?)\\.[^/.]+$` | no | the filename without its extension (each file is its own run) |\n\n## Upload modes [#upload-modes]\n\n* **`auto`**: files upload to cloud storage immediately after run detection. From watcher 1.2.1, the watcher also uploads files requested from the run page, and retries an upload that didn’t finish on its next start (up to three starts).\n* **`manual`**: runs are reported without uploading; the server decides which files to upload via a queue the watcher checks once a minute. Useful when uploads need human approval. See [Manage the watcher fleet → Approving manual uploads](/docs/manage-watchers#approving-manual-uploads).\n\nFiles of 100 MB or more upload in 64 MB parts from watcher 1.2.0, which is also the first version that can upload files larger than 5 GB.\n\n## Initial scan [#initial-scan]\n\nThe first time an environment is entered (via `init` or `config set-environment`),\n`initial_scan` decides what happens to files already in the watch directory. It\ndefaults by environment:\n\n* `production` → `full`: the existing backlog is uploaded (production is the source of truth).\n* `staging` / `preview` → `new-only`: the on-disk files are recorded as a baseline and skipped; only files created afterwards are uploaded.\n\nThis is why a fresh `init` on `staging`/`preview` doesn’t upload a PC’s history. Set\n`initial_scan: full` to opt back in (e.g. to deliberately populate staging), or\n`initial_scan: new-only` on production to suppress its backlog. Upgrading an\nexisting watcher is unaffected: the environment’s local DB already carries history,\nso baseline seeding is skipped.\n\n## Switching environments [#switching-environments]\n\nA single PC can move between `staging`, `production`, and `preview`. Because these\nare separate deployments with separate databases, each keeps its own registration\nin `watcher_ids` and its own credentials in `~/.data-hub/.env.`.\n\n```sh\n# Switch to staging (reuses a stored registration, or registers one).\ndata-hub-watcher config set-environment staging\n\n# Point at a preview deployment (the base URL is required).\ndata-hub-watcher config set-environment preview \\\n --api-base-url https://data-hub-git-my-branch.vercel.app/api/v1\n```\n\nA switch validates the target API, reuses (or registers) the watcher id for that\nenvironment, pushes the config, and leaves a breadcrumb event in the old\nenvironment. A running `watch` process or service keeps using the old\nenvironment until restarted: on Windows, `service stop && service start`;\nelsewhere, restart `watch`.\n" + }, + { + "page": "cli-reference", + "title": "Watcher CLI", + "description": "Every data-hub-watcher command, subcommand, and flag in one place: init, watch, upload, config, service, state, and self-update.", + "markdown": "# Watcher CLI (/docs/cli-reference)\n\n\n\n**For:** Lab operators\n\nFlags and subcommands for `data-hub-watcher`, generated from the Click CLI in the watcher package. For task walkthroughs see [Install and update the watcher](/docs/install-the-watcher), [Run as a Windows service](/docs/windows-service), and [Roll out watcher releases](/docs/watcher-releases).\n\n## Command reference [#command-reference]\n\nThe catalog below lists every command, argument, and option from the installed CLI help text.\n\n## Global flags\n\nThese options apply to every `data-hub-watcher` command (watcher 1.2.1).\n\n**Flags**\n\n- `--config PATH` `path` env: `DATA_HUB_CONFIG_PATH` — Override config file path.\n- `--verbose` `boolean` — Enable debug logging.\n- `--version` `boolean` — Show the version and exit.\n\n## `data-hub-watcher config`\n\nView and manage the watcher config file.\n\n### `data-hub-watcher config edit`\n\nRe-prompt each config field with current values as defaults.\n\n### `data-hub-watcher config open`\n\nOpen the config file in your editor, then re-validate.\n\n**Flags**\n\n- `--editor` `text` — Editor command (e.g. --editor code).\n\n### `data-hub-watcher config path`\n\nPrint the resolved config file path.\n\n### `data-hub-watcher config set-environment`\n\nSwitch the watcher to a different API environment.\n\n**Arguments**\n\n- `ENVIRONMENT` (required) `choice (staging | production | preview)`\n\n**Flags**\n\n- `--api-base-url` `text` — API base URL for the target environment. Required the first time you switch to an environment; reused from config on later switches.\n- `--api-key` `text` — API key for the target environment.\n- `--show-key` `boolean` — Echo the API key as typed (useful on Windows terminals).\n- `--no-register` `boolean` — Fail instead of registering a new watcher if none is stored for the target env.\n\n### `data-hub-watcher config show`\n\nPretty-print the current config.\n\n### `data-hub-watcher config validate`\n\nValidate the config file (offline, no network calls).\n\n## `data-hub-watcher init`\n\nInteractive setup wizard + API registration.\n\n**Flags**\n\n- `--show-key` `boolean` — Echo the API key as it is typed/pasted. Useful on Windows terminals where hidden input is unreliable for paste.\n\n## `data-hub-watcher self-update`\n\nCheck the server for a newer watcher release and upgrade in place. Designed for unattended use — schedule via Windows Task Scheduler (e.g. weekly) so lab PCs converge on the latest published version without operator intervention.\n\n**Flags**\n\n- `--check` `boolean` — Print the server-reported target version without performing the upgrade.\n- `--force` `boolean` — Run the upgrade subprocess even if the local version already matches the target.\n\n## `data-hub-watcher service`\n\nWindows service management.\n\n### `data-hub-watcher service install`\n\nInstall the watcher as a Windows service.\n\n**Flags**\n\n- `--env-path FILE` `file` — Path to the .env file. Defaults to ~/.data-hub/.env..\n\n### `data-hub-watcher service reinstall`\n\nStop, uninstall, install, and start the watcher Windows service. Useful after upgrading the watcher wheel from an Administrator shell so the SCM picks up the new ``data_hub_watcher.service`` entrypoint without needing four separate ``service`` invocations.\n\n**Flags**\n\n- `--env-path FILE` `file` — Path to the .env file. Defaults to ~/.data-hub/.env..\n\n### `data-hub-watcher service start`\n\nStart the watcher Windows service.\n\n### `data-hub-watcher service status`\n\nShow the watcher Windows service status.\n\n### `data-hub-watcher service stop`\n\nStop the watcher Windows service.\n\n### `data-hub-watcher service uninstall`\n\nUninstall the watcher Windows service.\n\n## `data-hub-watcher state`\n\nInspect or reset the local upload history.\n\n### `data-hub-watcher state forget`\n\nForget upload history under a folder so those files are reported again. Clears every state database a watcher on this PC can open: the one `watch` uses and the one the Windows service uses. Stop the watcher first. It only scans the disk when it starts, so a running watcher will not pick the forgotten files back up.\n\n**Flags**\n\n- `--prefix` (required) `text` — Folder under the watch directory to forget, e.g. alice/.\n- `--yes` `boolean` — Skip the confirmation prompt.\n\n## `data-hub-watcher upload`\n\nUpload files to Data Hub (manual trigger). One-shot mode: pass both --file and --run-id to upload a single file. Queue mode: omit both to process the server-side upload queue.\n\n**Flags**\n\n- `--file PATH` `path` — Specific file to upload.\n- `--run-id` `text` — Associate upload with a specific run.\n- `--dry-run` `boolean` — Log what would be uploaded without uploading.\n\n## `data-hub-watcher watch`\n\nStart watching for new files.\n\n**Flags**\n\n- `--dry-run` `boolean` — Validate and log but don't start monitoring.\n\n## Interactive setup notes [#interactive-setup-notes]\n\n`init` and `config edit` prompt for values in the terminal. Those prompts are not CLI flags, so they do not appear in the catalog.\n\nKeys must start with `dhub_`. The watcher strips zero-width and non-breaking-space characters that some Windows clipboards inject. See [Install and update the watcher → Configure](/docs/install-the-watcher#configure).\n\n## Upload examples [#upload-examples]\n\nOne-shot upload outside the watch loop:\n\n```sh\ndata-hub-watcher upload --file /path/to/file.csv --run-id RUN001\ndata-hub-watcher upload\ndata-hub-watcher upload --dry-run\n```\n\nPass both `--file` and `--run-id`, or omit both to process the server-side queue in manual mode.\n\n## Switching environments [#switching-environments]\n\n`config set-environment` (and `config edit` when the environment changes) can take `--api-base-url`, `--api-key`, `--show-key`, and `--no-register`. Preview switches require `--api-base-url` the first time. See [Watcher configuration → Switching environments](/docs/configuration-reference#switching-environments).\n\n## Windows service notes [#windows-service-notes]\n\n`service` commands require Windows and the `windows-service` install extra. Use `--env-path` on `service install` and `service reinstall` when the service should load a non-default `.env` file. Full details are in [Run as a Windows service](/docs/windows-service).\n\n## Self-update examples [#self-update-examples]\n\n```sh\ndata-hub-watcher self-update\ndata-hub-watcher self-update --check\ndata-hub-watcher self-update --force\n```\n\nUpgrade behavior depends on how you installed the watcher. See [Install and update the watcher → Manual update](/docs/install-the-watcher#manual-update-self-update).\n" + }, + { + "page": "security", + "title": "Security and permissions", + "description": "Roles and what each can do, personal-access-token scopes, how authentication works, and good token hygiene.", + "markdown": "# Security and permissions (/docs/security)\n\n\n\n**For:** Administrators and developers\n\nThis page explains who can do what in Data Hub: the two roles, the scope system\nthat gates token-authenticated requests, and how to handle tokens safely. For the\nmechanics of creating and revoking tokens, see [Issue and revoke tokens](/docs/manage-tokens).\n\n## Authentication [#authentication]\n\nThe API accepts two kinds of credentials:\n\n* **Session cookies**: used by the web dashboard, which signs people in with Google OAuth via [Better Auth](https://www.better-auth.com). Session callers implicitly hold every scope; scope checks only apply to token requests.\n* **Bearer tokens**: personal access tokens (`dhub_…`) used by the watcher, the Lambda, and scripts, sent as `Authorization: Bearer `. A token acts as itself, under its own name; see [How tokens act](#how-tokens-act).\n* **OAuth access tokens**: issued to MCP clients after you sign in and approve a consent screen. Covered in [MCP access](#mcp-access).\n\nTokens are hashed with SHA-256 before storage; the plaintext is shown exactly once, at creation.\n\n### Who can sign in [#who-can-sign-in]\n\nAny Google account that finishes the sign-in flow becomes a member of your deployment. Data Hub itself checks nothing: there is no invite flow, no email allowlist, and no email-domain check, so a first successful sign-in creates the member record.\n\nRestrictions live on the Google side instead, in the OAuth client your team creates at deploy time. A consent screen set to **Internal** accepts only accounts in your Google Workspace organization. Set to **External**, it accepts any Google account, including personal ones. Engineers make that choice when they [create the OAuth client](https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/first-time-deployment.md#create-a-google-oauth-client).\n\nData Hub accepts 10 Google sign-in attempts per minute from one IP address and rejects the rest.\n\nEvery new member starts in the member role, never admin. See [First admin bootstrap](/docs/manage-members#first-admin-bootstrap) for how the first admin gets the role.\n\n### Session lifetime [#session-lifetime]\n\nSigning in stores a session in the database and sets a signed cookie. The session expires 7 days after its last refresh, and the dashboard refreshes it at most once a day, so a session in daily use doesn’t lapse on its own.\n\nThat cookie also carries a signed copy of the session for 5 minutes, which keeps most page loads off the database. Requests read the copy instead of the row. A change made in the database directly, such as deleting a session, therefore applies when the copy expires rather than on the next request.\n\nRevoking someone’s access on the Google side, by removing them from your Workspace organization for example, blocks their next sign-in but leaves any session already open working until it expires. Delete that user’s rows from the `session` table to cut them off sooner. Signing out from the sidebar ends the session immediately.\n\n## Roles [#roles]\n\n| Role | Can do |\n| ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |\n| **Member** | Sign in to the dashboard; view instruments, runs, files, and the token list; claim runs; comment, and edit or delete their own comments; delete and restore runs; send feedback and see the reports they sent. Cannot create or revoke tokens, confirm or update instruments, or manage other users. |\n| **Admin** | Everything members can, plus: confirm and update instruments; create and revoke tokens; promote and demote other users; delete any run comment; review every feedback report; manage **Settings → Integrations**; and set watcher release versions. |\n\nThe first admin is bootstrapped from the `ADMIN_EMAILS` environment variable\n(comma-separated, case-insensitive); listed users are promoted on every sign-in.\nAfter that, any admin can promote others from **Settings → Members**. Admins\ncannot demote themselves: that guards against locking the workspace out of its\nlast admin.\n\n### Admin-gated operations [#admin-gated-operations]\n\nA subset of mutations require the admin role in addition to (or instead of) a scope\ncheck:\n\n* `PATCH /api/v1/instruments/:id`: session callers must be admin (bearer-token automation still uses `instruments:write`, so watcher/Lambda flows are unaffected).\n* `POST /api/v1/tokens` and `DELETE /api/v1/tokens/:id`: admin-only, session-only. Bearer tokens cannot manage other tokens.\n* `GET /api/v1/users`, `PATCH /api/v1/users/:userId`: admin-only, session-only. Used by **Settings → Members**.\n* `/api/v1/settings/integrations/*`: admin-only, session-only. Used by **Settings → Integrations**.\n* `DELETE` on a run comment: admins can delete any comment; everyone else, including tokens, can delete only their own.\n* `GET /api/v1/feedback`: admins get every report; members get only the reports they sent.\n\n## How tokens act [#how-tokens-act]\n\nA personal access token acts as itself, not as the admin who created it. Data Hub records everything a token does under the token’s name, and the app shows that name with a key icon: on comments, deleted runs, retired instruments, removed watchers, and notifications sent through the API. A revoked token keeps its record, so its name stays visible and is marked **(revoked)**.\n\nBecause a token isn’t a person, three things need a signed-in person instead: claiming runs, sending feedback, and reviewing feedback. Tokens can’t do them, whatever their scopes.\n\nThe routes that the watcher and the Lambda call work the other way round. They accept only tokens and return `401 Unauthorized` to a browser session, so a signed-in member can’t act as a watcher. These routes cover watcher registration and heartbeats, upload links, file updates, run creation and updates, and instrument creation.\n\nTokens keep working when the admin who created them leaves. Revoke a departing person’s tokens yourself; see [When someone leaves](/docs/manage-tokens#when-someone-leaves).\n\n## Token scopes [#token-scopes]\n\nEvery personal access token carries an array of permission **scopes**. A token-authenticated request is rejected with `403 FORBIDDEN` when the token’s scopes don’t include the scope a route requires. Issue tokens with the least privilege they need. New tokens must list fine-grained scopes; `POST /api/v1/tokens` rejects the wildcard `*` and the older coarse scopes `runs:write`, `files:write`, and `watchers:write`. Tokens that already carry those still work until you rotate them.\n\n| Scope | Grants |\n| ---------------------- | ---------------------------------------------------------------------------------- |\n| `instruments:read` | Read instruments, dashboard status, and file patterns |\n| `instruments:write` | Create instruments and edit their configuration |\n| `runs:read` | Read and search runs, their files, comments, and attributions |\n| `runs:create` | Create run records |\n| `runs:update` | Update run metadata |\n| `runs:delete` | Soft-delete and restore runs |\n| `runs:reprocess` | Re-run the processing workflow for a run |\n| `runs:upload` | Request presigned S3 URLs to upload files to a run |\n| `runs:comment` | Post run comments as this token, and edit or delete the ones it posted |\n| `files:read` | Read file metadata and download files and run archives |\n| `files:create` | Register file records against a run |\n| `files:update` | Update file metadata and upload state |\n| `files:delete` | Delete files |\n| `files:reprocess` | Re-run the processing workflow for a file |\n| `watchers:read` | Read watcher status, heartbeats, and upload queues |\n| `watchers:report` | Register, heartbeat, send events, and push watcher config |\n| `watchers:admin` | Delete watchers |\n| `archive-jobs:read` | Read run-archive job status |\n| `archive-jobs:write` | Update run-archive job status (Lambda callback) |\n| `notifications:create` | Send notifications to people, optionally about a run |\n| `*` | Matches every scope. Reserved for legacy backfilled tokens; rejected on new tokens |\n\nA watcher token typically needs `instruments:read`, `instruments:write`, `watchers:read`, `watchers:report`, `runs:create`, `runs:update`, `runs:upload`, and `files:update`. That is what the **Watcher** preset selects. The **Lambda** preset selects `instruments:read`, `runs:read`, `runs:create`, `runs:update`, `files:create`, `files:update`, and `archive-jobs:write`. A read-only analysis token needs the `:read` scopes for the data you want to query.\n\nThe `runs:attribute` and `feedback:admin` scopes no longer exist, because claiming runs and reviewing feedback need a signed-in person. They were removed from existing tokens on October 6, 2026.\n\n## Integration secrets [#integration-secrets]\n\nThe Linear and Slack secrets that admins save under **Settings → Integrations** are encrypted in the database with the `INTEGRATION_SECRETS_KEY` environment variable, and the app never shows them again after saving. See [Set up integrations](/docs/integrations#integration-secrets-key).\n\n## MCP access [#mcp-access]\n\nThe MCP server at `/mcp/v1` doesn’t accept personal access tokens. An AI client runs an OAuth flow instead: you sign in to Data Hub, approve a consent screen that names the client and the access it wants, and the client receives an access token valid for that deployment’s MCP endpoint alone. A token issued to a client with no person behind it is rejected, so every MCP action attributes to a user.\n\nMCP recognizes two scopes rather than the `:` pairs above:\n\n* `read`: required to connect, and covers every read-only tool, resource, and prompt\n* `write`: additionally required by any tool that changes data, such as claiming a run or deleting one\n\nA grant belongs to one client and one person, and it carries that person’s own permissions. Clients register themselves at connect time, which creates an OAuth client record but grants nothing on its own: Google sign-in still decides who gets through, and the consent screen still decides what they hand over.\n\nThe dashboard has no screen for reviewing or revoking MCP grants yet. Removing someone from your Google Workspace organization blocks new grants; cutting off a client that already holds a token means deleting the matching rows from the `oauthAccessToken` and `oauthRefreshToken` tables.\n\nClient setup is in [MCP overview](/docs/mcp).\n\n## Token hygiene [#token-hygiene]\n\n* **Name tokens descriptively** (e.g. “FPLC watcher, Lab 201”) so you can tell what each one is for.\n* **Set expirations** for temporary setups.\n* **Scope to least privilege**: don’t hand out broad tokens for a single read-only integration.\n* **Revoke immediately** when a watcher is decommissioned or a token may have been exposed. Revocation is instant; clients using the token start getting `401 Unauthorized`. See [After revoking a token](/docs/manage-tokens#after-revoking-a-token).\n* **Revoke a departing person’s tokens.** Tokens don’t stop working when the person who created them leaves. Check the **Created by** column under **Settings → Access Tokens**.\n\n## Public pages, gated bodies [#public-pages-gated-bodies]\n\nSome web pages (dashboard, instruments, run detail, settings) are reachable without\na session so link unfurlers can read page metadata. The page body renders a\nsign-in prompt instead of real data when there’s no session, and three independent\nlayers (`robots` metadata, `robots.txt`, and an `X-Robots-Tag` header) keep the\nproduct app out of search indexes. The `/api/v1/*` surface always requires a\nsession cookie or bearer token.\n" + } + ] +} diff --git a/web/lib/mcp/docs/markdown.ts b/web/lib/mcp/docs/markdown.ts new file mode 100644 index 00000000..142a0972 --- /dev/null +++ b/web/lib/mcp/docs/markdown.ts @@ -0,0 +1,178 @@ +// Text helpers for the docs bundle: splitting a page into sections, flattening +// Markdown for indexing, and cutting excerpts for search results. + +const HEADING = /^(#{2,3})\s+(.+?)\s*$/; +// The docs pin anchors with a trailing `[#id]`; headings without one get the +// slug the docs site generates. +const EXPLICIT_ID = /\s*\[#([^\]\s]+)\]\s*$/; +const FENCE = /^\s*(`{3,}|~{3,})/; + +export interface MarkdownSection { + heading: string; + id: string; + level: 2 | 3; + /** Heading plus everything beneath it, including deeper headings. */ + markdown: string; + /** Text up to the next heading of any level, so search hits stay specific. */ + text: string; +} + +export interface SplitMarkdown { + /** Text before the first `##` heading, minus the page's `#` title line. */ + intro: string; + sections: MarkdownSection[]; +} + +export function slugifyHeading(text: string): string { + return text + .toLowerCase() + .replace(/[^\p{L}\p{N}\p{M}_ -]/gu, "") + .replace(/ /g, "-"); +} + +interface HeadingMark { + heading: string; + id: string; + index: number; + level: 2 | 3; +} + +function findHeadings(lines: string[]): HeadingMark[] { + const headings: HeadingMark[] = []; + const seen = new Map(); + let fence: string | null = null; + + for (const [index, line] of lines.entries()) { + const marker = FENCE.exec(line)?.[1]; + if (fence) { + if (marker?.startsWith(fence)) { + fence = null; + } + continue; + } + if (marker) { + fence = marker; + continue; + } + + const match = HEADING.exec(line); + if (!match) { + continue; + } + const explicit = EXPLICIT_ID.exec(match[2] ?? "")?.[1]; + const heading = (match[2] ?? "").replace(EXPLICIT_ID, "").trim(); + const base = explicit ?? slugifyHeading(heading); + const count = seen.get(base) ?? 0; + seen.set(base, count + 1); + headings.push({ + heading, + id: count === 0 ? base : `${base}-${count}`, + index, + level: match[1]?.length === 2 ? 2 : 3, + }); + } + return headings; +} + +export function splitSections(markdown: string): SplitMarkdown { + const lines = markdown.split("\n"); + const headings = findHeadings(lines); + + const first = headings[0]?.index ?? lines.length; + const intro = lines + .slice(0, first) + .filter((line, index) => !(index === 0 && line.startsWith("# "))) + .join("\n") + .trim(); + + const sections = headings.map((mark, position): MarkdownSection => { + const next = headings[position + 1]; + const end = headings + .slice(position + 1) + .find((later) => later.level <= mark.level); + return { + id: mark.id, + heading: mark.heading, + level: mark.level, + text: lines + .slice(mark.index + 1, next?.index ?? lines.length) + .join("\n") + .trim(), + markdown: lines + .slice(mark.index, end?.index ?? lines.length) + .join("\n") + .trim(), + }; + }); + + return { intro, sections }; +} + +const JSX_TAG = /<\/?[A-Z][A-Za-z]*(?:\s[^>]*?)?\/?>/g; +const JSX_TEXT_ATTRIBUTE = /\b(?:title|description)="([^"]*)"/g; + +/** + * Flattens Markdown to plain sentences for indexing and excerpts. Component + * tags such as `` carry real wording in + * their attributes, so those survive while the tags themselves go. + */ +export function toPlainText(markdown: string): string { + return markdown + .replace(JSX_TAG, (tag) => { + const phrases = [...tag.matchAll(JSX_TEXT_ATTRIBUTE)] + .map((match) => match[1] ?? "") + .filter(Boolean) + .map((phrase) => (/[.!?]$/.test(phrase) ? phrase : `${phrase}.`)); + return phrases.length > 0 ? ` ${phrases.join(" ")} ` : " "; + }) + .replace(/^\s*(`{3,}|~{3,}).*$/gm, " ") + .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + .replace(EXPLICIT_ID, "") + .replace(/^#{1,6}\s+/gm, "") + .replace(/^[ \t]*>[ \t]?/gm, "") + .replace(/^[ \t|:-]{3,}$/gm, " ") + .replace(/[`*]/g, "") + .replace(/\|/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +const EXCERPT_LENGTH = 300; +// Characters kept before the first match, so the excerpt reads as a sentence. +const EXCERPT_LEAD_IN = 80; + +/** + * A short passage around the first place a search term appears, trimmed to + * word boundaries. Falls back to the start of the text when nothing matches, + * for example when only the page title matched. + */ +export function makeExcerpt( + plainText: string, + terms: readonly string[] +): string { + if (plainText.length <= EXCERPT_LENGTH) { + return plainText; + } + + const lower = plainText.toLowerCase(); + const hits = terms + .map((term) => lower.indexOf(term.toLowerCase())) + .filter((position) => position >= 0); + const anchor = hits.length > 0 ? Math.min(...hits) : 0; + + let start = Math.max(0, anchor - EXCERPT_LEAD_IN); + if (start > 0) { + const boundary = plainText.indexOf(" ", start); + start = boundary === -1 ? start : boundary + 1; + } + let end = Math.min(plainText.length, start + EXCERPT_LENGTH); + if (end < plainText.length) { + const boundary = plainText.lastIndexOf(" ", end); + end = boundary > start ? boundary : end; + } + + return `${start > 0 ? "…" : ""}${plainText.slice(start, end).trim()}${ + end < plainText.length ? "…" : "" + }`; +} diff --git a/web/lib/mcp/docs/search.ts b/web/lib/mcp/docs/search.ts new file mode 100644 index 00000000..281cefcb --- /dev/null +++ b/web/lib/mcp/docs/search.ts @@ -0,0 +1,273 @@ +import MiniSearch from "minisearch"; +import { type DocEntry, type DocSection, getDocsBundle } from "./bundle"; +import { makeExcerpt } from "./markdown"; + +const SEARCH_RESULT_LIMIT = 5; +// Keeps the `SEARCH_RESULT_LIMIT` results from all being sections of one long +// page. +const MAX_HITS_PER_PAGE = 2; +// Changelog entries are short, so they would otherwise outrank the docs page +// that explains the same thing in more depth. +const CHANGELOG_WEIGHT = 0.7; + +// Questions arrive as sentences. Without this, "how do I" matches every page. +// `data` and `hub` are here because every page names the product. +const STOP_WORDS = new Set([ + "a", + "about", + "am", + "an", + "and", + "are", + "as", + "at", + "be", + "been", + "but", + "by", + "can", + "could", + "data", + "did", + "do", + "does", + "for", + "from", + "get", + "had", + "has", + "have", + "how", + "hub", + "i", + "if", + "in", + "into", + "is", + "it", + "its", + "me", + "much", + "my", + "myself", + "of", + "on", + "or", + "our", + "should", + "so", + "that", + "the", + "their", + "then", + "there", + "this", + "to", + "up", + "us", + "was", + "we", + "were", + "what", + "when", + "where", + "which", + "who", + "why", + "will", + "with", + "would", + "you", + "your", +]); + +// Strips plurals only, applied to the index and the query alike, so "tokens" +// finds "token" without a stemming library. +function normalizeTerm(term: string): string | null { + const lower = term.toLowerCase(); + if (STOP_WORDS.has(lower)) { + return null; + } + if (lower.length > 4 && lower.endsWith("ies")) { + return `${lower.slice(0, -3)}y`; + } + if (lower.length > 3 && lower.endsWith("s") && !lower.endsWith("ss")) { + return lower.slice(0, -1); + } + return lower; +} + +const tokenize: (text: string) => string[] = MiniSearch.getDefault("tokenize"); + +/** The question's words as the index stores them, mapped to how they were typed. */ +function queryWords(query: string): Map { + const words = new Map(); + for (const token of tokenize(query)) { + const term = normalizeTerm(token); + if (term && !words.has(term)) { + words.set(term, token); + } + } + return words; +} + +// Search matches any one word, so a result alone says little. A section that +// holds two thirds of the question's words, rounded up, counts as an answer. +// The golden questions in `tests/unit/mcp-docs-search.test.ts` check the bar. +function wordsNeeded(wordCount: number): number { + return Math.ceil((wordCount * 2) / 3); +} + +interface IndexedPassage { + description: string; + heading: string; + id: string; + kind: DocEntry["kind"]; + pageTitle: string; + text: string; +} + +interface Passage { + entry: DocEntry; + /** Text the excerpt is cut from. */ + excerptSource: string; + /** Null for the text before a page's first heading. */ + section: DocSection | null; +} + +function passagesOf( + entry: DocEntry +): Array<{ passage: Passage; indexed: IndexedPassage }> { + const description = entry.kind === "docs" ? entry.description : ""; + const parts: Array<{ + heading: string; + section: DocSection | null; + text: string; + }> = []; + if (entry.introText || entry.sections.length === 0) { + parts.push({ section: null, heading: entry.title, text: entry.introText }); + } + for (const section of entry.sections) { + parts.push({ section, heading: section.heading, text: section.plainText }); + } + + return parts.map(({ section, heading, text }) => ({ + passage: { entry, section, excerptSource: text || description }, + indexed: { + id: `${entry.page}#${section?.id ?? ""}`, + kind: entry.kind, + pageTitle: entry.title, + description, + heading, + text, + }, + })); +} + +interface DocsIndex { + bundle: DocEntry[]; + index: MiniSearch; + passages: Map; +} + +let cached: DocsIndex | null = null; + +function getIndex(): DocsIndex { + const bundle = getDocsBundle(); + // The bundle is the same array on every call in production, so the index is + // built once per server instance. Elsewhere it is rebuilt with the bundle. + if (cached?.bundle === bundle) { + return cached; + } + + const index = new MiniSearch({ + fields: ["heading", "pageTitle", "description", "text"], + storeFields: ["kind"], + processTerm: normalizeTerm, + searchOptions: { + boost: { heading: 3, pageTitle: 2, description: 1.5 }, + // Short terms such as "run" would otherwise prefix-match half the docs. + prefix: (term) => term.length >= 4, + fuzzy: (term) => (term.length >= 5 ? 0.2 : false), + boostDocument: (_id, _term, stored) => + stored?.kind === "changelog" ? CHANGELOG_WEIGHT : 1, + }, + }); + + const passages = new Map(); + const documents: IndexedPassage[] = []; + for (const entry of bundle) { + for (const { passage, indexed } of passagesOf(entry)) { + passages.set(indexed.id, passage); + documents.push(indexed); + } + } + index.addAll(documents); + + cached = { bundle, index, passages }; + return cached; +} + +export interface DocsHit { + entry: DocEntry; + excerpt: string; + section: DocSection | null; +} + +export interface DocsSearchOutcome { + /** + * Sections that answer the question by the `wordsNeeded` bar. Zero when a + * word appears nowhere in the docs, which usually means they don't cover it. + */ + goodMatches: number; + hits: DocsHit[]; + /** Words of the question that appear nowhere in the docs, as typed. */ + missingWords: string[]; +} + +// Matched terms are normalized (and plural-stripped), so cut a letter off long +// ones to still find the original word in the text. +function excerptTerms(terms: readonly string[]): string[] { + return terms.map((term) => (term.length > 4 ? term.slice(0, -1) : term)); +} + +export function searchDocs(query: string): DocsSearchOutcome { + const { index, passages } = getIndex(); + const words = queryWords(query); + const results = index.search(query); + + const found = new Set(results.flatMap((result) => result.queryTerms)); + const missingWords = [...words] + .filter(([term]) => !found.has(term)) + .map(([, typed]) => typed); + const needed = wordsNeeded(words.size); + const goodMatches = + missingWords.length > 0 + ? 0 + : results.filter((result) => result.queryTerms.length >= needed).length; + + const perPage = new Map(); + const hits: DocsHit[] = []; + for (const result of results) { + const passage = passages.get(result.id); + if (!passage) { + continue; + } + const used = perPage.get(passage.entry.page) ?? 0; + if (used >= MAX_HITS_PER_PAGE) { + continue; + } + perPage.set(passage.entry.page, used + 1); + + hits.push({ + entry: passage.entry, + section: passage.section, + excerpt: makeExcerpt(passage.excerptSource, excerptTerms(result.terms)), + }); + if (hits.length >= SEARCH_RESULT_LIMIT) { + break; + } + } + + return { goodMatches, hits, missingWords }; +} diff --git a/web/lib/mcp/instructions.ts b/web/lib/mcp/instructions.ts index aba8dc8e..11428d0c 100644 --- a/web/lib/mcp/instructions.ts +++ b/web/lib/mcp/instructions.ts @@ -1,46 +1,42 @@ -import { feedbackInstructions } from "@arcadiascience/app-feedback-toolkit/mcp"; -import { FEEDBACK_MCP_OPTIONS } from "@/lib/feedback-app"; +import { FEEDBACK_APP } from "@/lib/feedback-app"; /** - * Delivered to every MCP client at initialize — reachable without resources. - * - * Scoped to what tool descriptions cannot carry on their own: cross-tool - * constraints, routing, and volume limits. Full status and date definitions - * live in `datahub://glossary` so they are not restated here. + * Sent to every client at initialize. Claude Code keeps only the first 2 KB, so + * a unit test caps the length and the most important points come first. Holds + * only what tool descriptions can't: what the server is for, cross-tool rules, + * routing, and volume limits. Definitions live in the docs (`search_docs`). */ export const MCP_SERVER_INSTRUCTIONS = ` -Data Hub exposes lab instrument runs, their files, watcher agents, and run -attributions (who ran what). +Data Hub collects files from lab instruments, processes them, and groups them +into runs. This server covers instruments, runs, files, watchers (the program +on each instrument PC that uploads files), who ran what, and Data Hub's own +documentation. -Writes: mutating tools need the write scope and otherwise fail with "Token is -missing required scope: write" — ask the user to re-authorize rather than -retrying. Confirm with the user before deleting, dismissing, unclaiming, or -reprocessing anything. Attribution and comments always act as the token's -owner; you cannot claim a run or comment as someone else. +How-to questions about Data Hub itself (setup, watchers, tokens, permissions, +recent changes): call search_docs, then read_doc on the best match, and link +the page in your answer. -Dates: dateFrom/dateTo are inclusive UTC calendar days matched against -coalesce(acquired_at, created_at). The web dashboard uses the viewer's -timezone, so its daily counts can differ. +Writes need the write scope and otherwise fail with "Token is missing required +scope: write"; ask the user to re-authorize instead of retrying. Confirm before +deleting, dismissing, unclaiming, or reprocessing. Claims and comments always +act as the signed-in user. -Run status is derived from a run's file states, in this order of precedence: -failed > stalled > pending > uploaded > processing > completed > empty. -Stalled runs recover with reprocess_run; pending runs need -request_run_upload_all and an online watcher. Definitions: datahub://glossary. +Dates: dateFrom/dateTo are inclusive UTC days, matched on a run's acquired +time, or its created time when that is missing. The dashboard uses the +viewer's timezone, so its daily counts can differ. + +Stalled runs recover with reprocess_run. Pending runs need +request_run_upload_all and an online watcher. Tool routing: - global_search for filenames, instrument names, users, or comments. -- search_runs for date, status, or instrument-metadata filters. -- My runs: search_runs with ranBy="me" (or get_me then ranBy=). -- get_run_report for run results; it also renders an interactive report in - hosts that support MCP Apps. get_run returns metadata only. -- Metadata filter values: get_instrument_filter_options or - datahub://instruments/{id}/filter-options. -- Watcher diagnosis: list_watchers → get_watcher_heartbeats → - list_watcher_events (unhealthy agents only). - -A run can have thousands of files: filter list_run_files by status rather than -paging all of it, and prefer get_run_report's bounded sample over downloading -full CSVs. +- search_runs for date, status, or metadata filters; ranBy="me" for my runs. +- Metadata filter values: get_instrument_filter_options. +- get_run_report for results; get_run is metadata only. +- Watchers: list_watchers → get_watcher_heartbeats → list_watcher_events. +- Large runs: filter list_run_files by status, and prefer get_run_report's + sample over full CSVs. -${feedbackInstructions(FEEDBACK_MCP_OPTIONS)} +Feedback: send_feedback for bugs and requests about ${FEEDBACK_APP.appName} +(show the user the draft first). ${FEEDBACK_APP.mcpGuidance} `.trim(); diff --git a/web/lib/mcp/resources.defs.ts b/web/lib/mcp/resources.defs.ts index 8124823d..1d6f3c0c 100644 --- a/web/lib/mcp/resources.defs.ts +++ b/web/lib/mcp/resources.defs.ts @@ -40,6 +40,15 @@ export const instrumentFilterOptionsResource = { uriTemplate: "datahub://instruments/{instrumentId}/filter-options", } as const satisfies McpResourceDef; +export const docsPageResource = { + name: "docs-page", + description: + "A page of Data Hub's own documentation as Markdown, such as datahub://docs/manage-tokens. Same text as the read_doc tool. Use search_docs to find the right page.", + mimeType: "text/markdown", + kind: "template", + uriTemplate: "datahub://docs/{page}", +} as const satisfies McpResourceDef; + export const runReportUiResource = { name: "run-report", description: @@ -54,5 +63,6 @@ export const MCP_RESOURCE_DEFS = [ meResource, glossaryResource, instrumentFilterOptionsResource, + docsPageResource, runReportUiResource, ] as const; diff --git a/web/lib/mcp/resources.ts b/web/lib/mcp/resources.ts index a9a012e5..10e8b58a 100644 --- a/web/lib/mcp/resources.ts +++ b/web/lib/mcp/resources.ts @@ -3,6 +3,12 @@ import { getInstruments, getUserById } from "@/lib/api/dashboard"; import { getInstrumentListWithCounts } from "@/lib/api/instruments"; import type { InstrumentType } from "@/lib/db/schema"; import { completeInstrumentId } from "@/lib/mcp/completions"; +import { + findDoc, + listDocPages, + parseDocRef, + unknownPageMessage, +} from "@/lib/mcp/docs/bundle"; import { DATAHUB_GLOSSARY } from "@/lib/mcp/glossary"; import { resolveInstrumentFilterOptions } from "@/lib/mcp/instrument-filter-options"; import { loadRunReportHtml } from "@/lib/mcp/run-report-html"; @@ -13,6 +19,7 @@ import { } from "@/lib/mcp/ui-apps"; import { runReportUiMeta } from "@/lib/mcp/ui-csp"; import { + docsPageResource, glossaryResource, instrumentFilterOptionsResource, instrumentsResource, @@ -49,6 +56,22 @@ function filterOptionsDescription(instrumentType: InstrumentType): string { } } +// Clients such as Claude Desktop show only a resource's name in their attach +// menu, and three docs pages share the title "Overview". +function docsPageResourceList() { + const seen = new Set(); + return listDocPages().map(({ page, title, description }) => { + const name = seen.has(title) ? `${title} (${page})` : title; + seen.add(title); + return { + uri: `datahub://docs/${page}`, + name, + description, + mimeType: docsPageResource.mimeType, + }; + }); +} + export function registerResources(server: McpServer) { server.registerResource( instrumentsResource.name, @@ -203,6 +226,39 @@ export function registerResources(server: McpServer) { } ); + server.registerResource( + docsPageResource.name, + new ResourceTemplate(docsPageResource.uriTemplate, { + list: async () => ({ resources: docsPageResourceList() }), + complete: { + page: (value: string) => + listDocPages() + .map(({ page }) => page) + .filter((page) => page.startsWith(value.toLowerCase())), + }, + }), + { + description: docsPageResource.description, + mimeType: docsPageResource.mimeType, + }, + (uri, variables) => { + const requested = Array.isArray(variables.page) + ? variables.page[0] + : variables.page; + const { page } = parseDocRef(requested ?? ""); + const entry = findDoc(page); + return { + contents: [ + { + uri: uri.href, + mimeType: docsPageResource.mimeType, + text: entry ? entry.markdown : unknownPageMessage(page), + }, + ], + }; + } + ); + server.registerResource( runReportUiResource.name, runReportUiResource.uri, diff --git a/web/lib/mcp/tools/docs.defs.ts b/web/lib/mcp/tools/docs.defs.ts new file mode 100644 index 00000000..af9462f4 --- /dev/null +++ b/web/lib/mcp/tools/docs.defs.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; +import type { McpToolDef } from "@/lib/mcp/catalog/types"; +import { readDocOutputSchema, searchDocsOutputSchema } from "./docs.output"; + +export const searchDocsTool = { + name: "search_docs", + title: "Search Docs", + description: + "Search Data Hub's own documentation and changelog. Use it for how-to and explanation questions about Data Hub itself: setting up instruments and watchers, tokens, permissions, notifications, MCP and API access, and what changed recently. Returns up to five matching sections, each with a short excerpt, a page ID, and a link to cite. Call read_doc to read the full page or section. Not for lab data; use global_search or search_runs for runs, files, and instruments. When nothing matches, the result lists every docs page so you can pick one. If the results do not answer the question, say the docs do not cover it and offer to report the gap with send_feedback.", + group: "docs", + inputSchema: { + query: z + .string() + .describe( + "What to look up, as a question or keywords (min 2 characters)." + ), + }, + outputSchema: searchDocsOutputSchema, + annotations: { readOnlyHint: true }, +} as const satisfies McpToolDef; + +export const readDocTool = { + name: "read_doc", + title: "Read Doc", + description: + "Read one page of Data Hub's documentation, or one changelog entry, as Markdown. Pass a page ID from search_docs, such as `manage-tokens` or `changelog/2026-10-08-hina-large-image-previews`. Add `section` (a heading ID from the search result) to read only that part. The result lists the page's section IDs, so a long page can be read in pieces. Cite the returned link in your answer.", + group: "docs", + inputSchema: { + page: z + .string() + .describe( + "Page ID from search_docs. A docs path or URL such as /docs/manage-tokens is also accepted." + ), + section: z + .string() + .optional() + .describe("Heading ID within the page. Omit to read the whole page."), + }, + outputSchema: readDocOutputSchema, + annotations: { readOnlyHint: true }, +} as const satisfies McpToolDef; + +export const DOCS_TOOL_DEFS = [searchDocsTool, readDocTool] as const; diff --git a/web/lib/mcp/tools/docs.output.ts b/web/lib/mcp/tools/docs.output.ts new file mode 100644 index 00000000..05bf8983 --- /dev/null +++ b/web/lib/mcp/tools/docs.output.ts @@ -0,0 +1,63 @@ +import { z } from "zod"; + +const docPageSummarySchema = z.object({ + page: z.string(), + title: z.string(), + description: z.string(), +}); + +const docsSearchResultSchema = z.object({ + page: z + .string() + .describe("Page ID to pass to read_doc (`changelog/` for a change)."), + title: z.string(), + section: z + .string() + .nullable() + .describe("Heading ID of the matching section, or null for the intro."), + heading: z.string().nullable(), + excerpt: z.string(), + url: z + .string() + .nullable() + .describe("Link to cite. Null for a change that has no docs page."), + date: z + .string() + .nullable() + .describe("YYYY-MM-DD the change shipped. Null for a docs page."), +}); + +export const searchDocsOutputSchema = z.object({ + results: z.array(docsSearchResultSchema), + pages: z + .array(docPageSummarySchema) + .optional() + .describe("Every docs page. Present only when nothing matched."), + hint: z + .string() + .optional() + .describe( + "Present when no section matches most of the question. Says what to do next." + ), +}); + +export const readDocOutputSchema = z.object({ + page: z.string(), + title: z.string(), + section: z.string().nullable(), + heading: z.string().nullable(), + url: z.string().nullable(), + markdown: z.string(), + sections: z + .array( + z.object({ + id: z.string(), + heading: z.string(), + level: z + .number() + .int() + .describe("2 for a section, 3 for a subsection."), + }) + ) + .describe("Heading IDs on the page, for reading one section at a time."), +}); diff --git a/web/lib/mcp/tools/docs.ts b/web/lib/mcp/tools/docs.ts new file mode 100644 index 00000000..1226ac6c --- /dev/null +++ b/web/lib/mcp/tools/docs.ts @@ -0,0 +1,122 @@ +import type { McpServer } from "@modelcontextprotocol/server"; +import { appOrigin } from "@/lib/app-origin"; +import { DOCS_URL } from "@/lib/docs"; +import { trackDocsRead, trackDocsSearch } from "@/lib/mcp/analytics"; +import { toolRegistrationConfig } from "@/lib/mcp/catalog/register"; +import { + type DocEntry, + findDoc, + listDocPages, + parseDocRef, + unknownPageMessage, +} from "@/lib/mcp/docs/bundle"; +import { searchDocs } from "@/lib/mcp/docs/search"; +import { errorResult, structuredResult } from "@/lib/mcp/tools/helpers"; +import { MIN_QUERY_LENGTH } from "@/lib/search-constants"; +import { readDocTool, searchDocsTool } from "./docs.defs"; + +const REPORT_GAP = + "If the docs do not cover the question, say so and offer to report the gap with send_feedback."; + +const NO_MATCH_HINT = `Nothing matched. Pick a page from \`pages\` and call read_doc, or try different words. ${REPORT_GAP}`; + +function weakMatchHint(missingWords: readonly string[]): string { + const missing = + missingWords.length > 0 + ? `The docs never mention ${missingWords.map((word) => `'${word}'`).join(", ")}. ` + : ""; + return `${missing}No section matches most of the question, so these results may not answer it. ${REPORT_GAP}`; +} + +// `DOCS_URL` is path-only when the docs share the app's origin, and an agent +// needs a link it can hand to a person. +function docsBaseUrl(): string { + return DOCS_URL.startsWith("http") ? DOCS_URL : `${appOrigin()}${DOCS_URL}`; +} + +function docUrl(entry: DocEntry, sectionId: string | null): string | null { + if (entry.kind === "changelog") { + return entry.docsUrl; + } + return `${docsBaseUrl()}/${entry.page}${sectionId ? `#${sectionId}` : ""}`; +} + +export function registerDocsTools(server: McpServer) { + server.registerTool( + searchDocsTool.name, + toolRegistrationConfig(searchDocsTool), + ({ query }, ctx) => { + const trimmed = query.trim(); + if (trimmed.length < MIN_QUERY_LENGTH) { + return errorResult( + `Query must be at least ${MIN_QUERY_LENGTH} characters.` + ); + } + + const { hits, goodMatches, missingWords } = searchDocs(trimmed); + trackDocsSearch(ctx, goodMatches); + + if (hits.length === 0) { + return structuredResult({ + results: [], + pages: listDocPages(), + hint: NO_MATCH_HINT, + }); + } + return structuredResult({ + results: hits.map(({ entry, section, excerpt }) => ({ + page: entry.page, + title: entry.title, + section: section?.id ?? null, + heading: section?.heading ?? null, + excerpt, + url: docUrl(entry, section?.id ?? null), + date: entry.kind === "changelog" ? entry.date : null, + })), + ...(goodMatches === 0 ? { hint: weakMatchHint(missingWords) } : {}), + }); + } + ); + + server.registerTool( + readDocTool.name, + toolRegistrationConfig(readDocTool), + ({ page, section }, ctx) => { + const ref = parseDocRef(page, section); + const entry = findDoc(ref.page); + if (!entry) { + return errorResult(unknownPageMessage(ref.page)); + } + + const wanted = ref.section?.toLowerCase(); + const match = wanted + ? entry.sections.find((s) => s.id.toLowerCase() === wanted) + : undefined; + if (wanted && !match) { + const ids = entry.sections.map((s) => s.id).join(", "); + return errorResult( + `Page '${entry.page}' has no section '${ref.section}'.${ + ids + ? ` Sections: ${ids}.` + : " It has no sections; read the whole page." + }` + ); + } + + trackDocsRead(ctx, entry.page); + return structuredResult({ + page: entry.page, + title: entry.title, + section: match?.id ?? null, + heading: match?.heading ?? null, + url: docUrl(entry, match?.id ?? null), + markdown: match ? match.markdown : entry.markdown, + sections: entry.sections.map(({ id, heading, level }) => ({ + id, + heading, + level, + })), + }); + } + ); +} diff --git a/web/lib/mcp/tools/index.ts b/web/lib/mcp/tools/index.ts index e5b8e26a..920be571 100644 --- a/web/lib/mcp/tools/index.ts +++ b/web/lib/mcp/tools/index.ts @@ -2,6 +2,7 @@ import { registerFeedbackTools } from "@arcadiascience/app-feedback-toolkit/mcp" import type { McpServer } from "@modelcontextprotocol/server"; import { feedback } from "@/lib/feedback"; import { registerDiscoveryTools } from "@/lib/mcp/tools/discovery"; +import { registerDocsTools } from "@/lib/mcp/tools/docs"; import { registerFileTools } from "@/lib/mcp/tools/files"; import { registerInstrumentTools } from "@/lib/mcp/tools/instruments"; import { registerReportViewTools } from "@/lib/mcp/tools/report-views"; @@ -13,6 +14,7 @@ export function registerTools(server: McpServer) { registerRunTools(server); registerReportViewTools(server); registerDiscoveryTools(server); + registerDocsTools(server); registerWatcherTools(server); registerFileTools(server); registerFeedbackTools(server, feedback); diff --git a/web/package-lock.json b/web/package-lock.json index b6266696..82f1153c 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -37,6 +37,7 @@ "drizzle-orm": "^0.45.2", "lucide-react": "^1.7.0", "mcp-handler": "^2.1.1", + "minisearch": "^7.2.0", "next": "^16.4.0", "next-themes": "^0.4.6", "nuqs": "^2.8.9", @@ -12095,6 +12096,12 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/minisearch": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz", + "integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", diff --git a/web/package.json b/web/package.json index ccf3a373..27c9af9a 100644 --- a/web/package.json +++ b/web/package.json @@ -61,6 +61,7 @@ "drizzle-orm": "^0.45.2", "lucide-react": "^1.7.0", "mcp-handler": "^2.1.1", + "minisearch": "^7.2.0", "next": "^16.4.0", "next-themes": "^0.4.6", "nuqs": "^2.8.9", diff --git a/web/tests/integration/mcp.test.ts b/web/tests/integration/mcp.test.ts index 6b8515aa..17912329 100644 --- a/web/tests/integration/mcp.test.ts +++ b/web/tests/integration/mcp.test.ts @@ -13,7 +13,7 @@ import { } from "@/tests/integration/helpers"; /** Bump when adding/removing an MCP tool so the change shows up in review. */ -const EXPECTED_MCP_TOOL_COUNT = 37; +const EXPECTED_MCP_TOOL_COUNT = 39; function jsonRpc(method: string, params: unknown = {}, id = 1) { return { @@ -197,6 +197,8 @@ describe("MCP Server (HTTP)", () => { expect(toolNames).toContain("list_watcher_events"); expect(toolNames).toContain("claim_runs"); expect(toolNames).toContain("get_instrument_filter_options"); + expect(toolNames).toContain("search_docs"); + expect(toolNames).toContain("read_doc"); expect(toolNames).toHaveLength(EXPECTED_MCP_TOOL_COUNT); }); @@ -222,6 +224,42 @@ describe("MCP Server (HTTP)", () => { ); }); + // ---- Docs tools (end-to-end) --------------------------------------------- + + // Checks the docs bundle is compiled into the production server. The + // changelog files still come from the source tree under `next start`. + it("search_docs and read_doc answer from the built app", async () => { + const search = await callTool("search_docs", { + query: "how do I revoke a token", + }); + expect(search.isError).toBeFalsy(); + const results = JSON.parse(search.content[0].text) as { + results: Array<{ page: string; section: string | null }>; + }; + expect(results.results.map((r) => r.page)).toContain("manage-tokens"); + + const read = await callTool("read_doc", { + page: "manage-tokens", + section: "revoke-a-token", + }); + expect(read.isError).toBeFalsy(); + const doc = JSON.parse(read.content[0].text) as { markdown: string }; + expect(doc.markdown).toContain("## Revoke a token"); + }); + + it("search_docs finds changelog entries in the built app", async () => { + const result = await callTool("search_docs", { + query: "Hina large image preview smaller", + }); + expect(result.isError).toBeFalsy(); + const parsed = JSON.parse(result.content[0].text) as { + results: Array<{ page: string }>; + }; + expect(parsed.results.some((r) => r.page.startsWith("changelog/"))).toBe( + true + ); + }); + // ---- Attribution tools (end-to-end) -------------------------------------- // These tests exercise the auth wiring: `authInfo.extra.userId` is the only diff --git a/web/tests/mcp/mcp-catalog.test.ts b/web/tests/mcp/mcp-catalog.test.ts index 13f9bfa6..bb174408 100644 --- a/web/tests/mcp/mcp-catalog.test.ts +++ b/web/tests/mcp/mcp-catalog.test.ts @@ -33,7 +33,7 @@ describe("MCP catalog document", () => { it("publishes outputSchema for every tool", () => { const doc = buildMcpCatalogDocument(); - expect(MCP_TOOL_DEFS).toHaveLength(37); + expect(MCP_TOOL_DEFS).toHaveLength(39); for (const def of MCP_TOOL_DEFS) { expect(def.outputSchema, `${def.name} missing outputSchema`).toBeTruthy(); const tool = doc.tools.find((t) => t.name === def.name); diff --git a/web/tests/mcp/mcp-protocol.test.ts b/web/tests/mcp/mcp-protocol.test.ts index ea6e9fd5..988ec57e 100644 --- a/web/tests/mcp/mcp-protocol.test.ts +++ b/web/tests/mcp/mcp-protocol.test.ts @@ -831,7 +831,7 @@ describe("MCP Protocol (in-memory)", () => { it("every registered tool advertises outputSchema", async () => { const { tools } = await client.listTools(); - expect(tools).toHaveLength(37); + expect(tools).toHaveLength(39); for (const tool of tools) { const schema = tool.outputSchema as | { type?: string; oneOf?: unknown; anyOf?: unknown } @@ -1056,6 +1056,164 @@ describe("MCP Protocol (in-memory)", () => { expect(result.isError).toBe(true); }); + it("search_docs returns matching sections with links and excerpts", async () => { + const result = await client.callTool({ + name: "search_docs", + arguments: { query: "how do I revoke a token" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + results: Array<{ + page: string; + section: string | null; + excerpt: string; + url: string | null; + }>; + pages?: unknown; + hint?: string; + }; + expect(result.structuredContent).toEqual(parsed); + expect(parsed.results.length).toBeGreaterThan(0); + expect(parsed.results.length).toBeLessThanOrEqual(5); + const revoke = parsed.results.find( + (r) => r.page === "manage-tokens" && r.section === "revoke-a-token" + ); + expect(revoke?.url).toMatch(/\/docs\/manage-tokens#revoke-a-token$/); + expect(revoke?.excerpt).toBeTruthy(); + expect(parsed.pages).toBeUndefined(); + expect(parsed.hint).toBeUndefined(); + }); + + it("search_docs flags results that don't answer the question", async () => { + const result = await client.callTool({ + name: "search_docs", + arguments: { query: "how do I export runs to Benchling" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + results: unknown[]; + pages?: unknown; + hint?: string; + }; + expect(parsed.results.length).toBeGreaterThan(0); + expect(parsed.pages).toBeUndefined(); + expect(parsed.hint).toContain("never mention 'Benchling'"); + expect(parsed.hint).toContain("send_feedback"); + }); + + it("search_docs lists every docs page when nothing matches", async () => { + const result = await client.callTool({ + name: "search_docs", + arguments: { query: "zxqv plorgnak" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + results: unknown[]; + pages: Array<{ page: string; title: string; description: string }>; + hint: string; + }; + expect(parsed.results).toEqual([]); + expect(parsed.pages.map((p) => p.page)).toContain("manage-tokens"); + expect(parsed.pages.length).toBeGreaterThan(20); + expect(parsed.hint).toContain("send_feedback"); + }); + + it("search_docs rejects short queries", async () => { + const result = await client.callTool({ + name: "search_docs", + arguments: { query: " a " }, + }); + expect(result.isError).toBe(true); + }); + + it("read_doc returns a whole page with its section IDs", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { page: "manage-tokens" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + page: string; + section: string | null; + url: string; + markdown: string; + sections: Array<{ id: string }>; + }; + expect(result.structuredContent).toEqual(parsed); + expect(parsed.page).toBe("manage-tokens"); + expect(parsed.section).toBeNull(); + expect(parsed.url).toMatch(/\/docs\/manage-tokens$/); + expect(parsed.markdown).toContain("## Revoke a token"); + expect(parsed.sections.map((s) => s.id)).toContain("revoke-a-token"); + }); + + it("read_doc returns only the requested section", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { page: "manage-tokens", section: "revoke-a-token" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + section: string; + heading: string; + url: string; + markdown: string; + }; + expect(parsed.section).toBe("revoke-a-token"); + expect(parsed.heading).toBe("Revoke a token"); + expect(parsed.url).toMatch(/\/docs\/manage-tokens#revoke-a-token$/); + expect(parsed.markdown.startsWith("## Revoke a token")).toBe(true); + expect(parsed.markdown).not.toContain("## Create a token"); + }); + + it("read_doc accepts a docs URL with a fragment", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { + page: "https://datahub.example.com/docs/manage-tokens#revoke-a-token", + }, + }); + expect(result.isError).toBeFalsy(); + expect((parseText(result.content) as { section: string }).section).toBe( + "revoke-a-token" + ); + }); + + it("read_doc reads a changelog entry", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { page: "changelog/2026-10-08-hina-large-image-previews" }, + }); + expect(result.isError).toBeFalsy(); + const parsed = parseText(result.content) as { + markdown: string; + url: string | null; + }; + expect(parsed.markdown).toContain("Hina"); + expect(parsed.url).toContain("/docs/instrument-preprocessing"); + }); + + it("read_doc names the pages it has when the page is unknown", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { page: "manage-token" }, + }); + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0]?.text ?? ""; + expect(text).toContain("Closest: manage-tokens"); + expect(text).toContain("Docs pages:"); + }); + + it("read_doc names the sections it has when the section is unknown", async () => { + const result = await client.callTool({ + name: "read_doc", + arguments: { page: "manage-tokens", section: "rotate" }, + }); + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text: string }>)[0]?.text ?? ""; + expect(text).toContain("revoke-a-token"); + }); + it("get_me errors without auth on the in-memory transport", async () => { const result = await client.callTool({ name: "get_me", @@ -1773,6 +1931,53 @@ describe("MCP Protocol (in-memory)", () => { ).toBe(true); }); + it("lists a resource template for docs pages", async () => { + const { resourceTemplates } = await client.listResourceTemplates(); + expect( + resourceTemplates.some((t) => t.uriTemplate === "datahub://docs/{page}") + ).toBe(true); + }); + + it("lists each docs page, but not the changelog, with a distinct name", async () => { + const { resources } = await client.listResources(); + const docs = resources.filter((r) => r.uri.startsWith("datahub://docs/")); + expect(docs.map((r) => r.uri)).toContain("datahub://docs/manage-tokens"); + expect(docs.some((r) => r.uri.includes("changelog"))).toBe(false); + expect(docs.every((r) => r.mimeType === "text/markdown")).toBe(true); + const names = docs.map((r) => r.name); + expect(new Set(names).size).toBe(names.length); + expect(names).toEqual( + expect.arrayContaining(["Overview", "Overview (api)", "Overview (mcp)"]) + ); + }); + + it("reads a docs page resource as Markdown", async () => { + const { contents } = await client.readResource({ + uri: "datahub://docs/manage-tokens", + }); + expect(contents[0]?.mimeType).toBe("text/markdown"); + expect((contents[0] as { text: string }).text).toContain( + "## Revoke a token" + ); + }); + + it("explains an unknown docs page resource", async () => { + const { contents } = await client.readResource({ + uri: "datahub://docs/no-such-page", + }); + expect((contents[0] as { text: string }).text).toContain("not found"); + }); + + it("completes docs page IDs", async () => { + const result = await client.complete({ + ref: { type: "ref/resource", uri: "datahub://docs/{page}" }, + argument: { name: "page", value: "manage-" }, + }); + expect(result.completion.values).toEqual( + expect.arrayContaining(["manage-tokens", "manage-members"]) + ); + }); + it("filter-options list surfaces both plate-reader and gel-doc instruments", async () => { const { resources } = await client.listResources(); const filterOptionUris = resources diff --git a/web/tests/unit/mcp-analytics.test.ts b/web/tests/unit/mcp-analytics.test.ts index 4f9d4062..9d187b2e 100644 --- a/web/tests/unit/mcp-analytics.test.ts +++ b/web/tests/unit/mcp-analytics.test.ts @@ -31,7 +31,12 @@ vi.mock("@/lib/mcp/client-name", async (importOriginal) => { }; }); -import { trackMcpConnect, withMcpTracking } from "@/lib/mcp/analytics"; +import { + trackDocsRead, + trackDocsSearch, + trackMcpConnect, + withMcpTracking, +} from "@/lib/mcp/analytics"; const authInfo: AuthInfo = { token: "token", @@ -313,3 +318,45 @@ describe("trackMcpConnect", () => { expect(trackEvent).not.toHaveBeenCalled(); }); }); + +describe("docs analytics", () => { + beforeEach(() => { + trackEvent.mockReset(); + mcpClientLabel.mockClear(); + }); + + it("records a docs search as a result bucket and nothing else", async () => { + trackDocsSearch(httpCtx(), 0); + expect(trackEvent.mock.calls[0]?.[0]).toBe("mcp_docs_search"); + await expect(lastProps()).resolves.toEqual({ + user_id: "user-1", + client: "Cursor", + result_bucket: "0", + }); + + trackDocsSearch(httpCtx(), 7); + await expect(lastProps()).resolves.toMatchObject({ + result_bucket: "1_to_10", + }); + trackDocsSearch(httpCtx(), 40); + await expect(lastProps()).resolves.toMatchObject({ + result_bucket: "over_10", + }); + }); + + it("records the page ID of a docs read", async () => { + trackDocsRead(httpCtx(), "manage-tokens"); + expect(trackEvent.mock.calls[0]?.[0]).toBe("mcp_docs_read"); + await expect(lastProps()).resolves.toEqual({ + user_id: "user-1", + client: "Cursor", + page: "manage-tokens", + }); + }); + + it("records nothing when no user is signed in", () => { + trackDocsSearch({ http: {} }, 3); + trackDocsRead({ http: {} }, "manage-tokens"); + expect(trackEvent).not.toHaveBeenCalled(); + }); +}); diff --git a/web/tests/unit/mcp-docs-markdown.test.ts b/web/tests/unit/mcp-docs-markdown.test.ts new file mode 100644 index 00000000..14cba9c8 --- /dev/null +++ b/web/tests/unit/mcp-docs-markdown.test.ts @@ -0,0 +1,245 @@ +import { describe, expect, it } from "vitest"; +import { + findDoc, + parseDocRef, + unknownPageMessage, +} from "@/lib/mcp/docs/bundle"; +import { + makeExcerpt, + slugifyHeading, + splitSections, + toPlainText, +} from "@/lib/mcp/docs/markdown"; + +const PAGE = [ + "# Manage tokens (/docs/manage-tokens)", + "", + "Intro line.", + "", + "## Create a token [#create-a-token]", + "", + "Open Settings.", + "", + "### Scope presets", + "", + "Pick a preset.", + "", + "```bash", + "# not a heading", + "## also not a heading", + "```", + "", + "## Revoke a token", + "", + "Click delete.", +].join("\n"); + +describe("splitSections", () => { + const { intro, sections } = splitSections(PAGE); + + it("keeps the text before the first heading as the intro, without the title", () => { + expect(intro).toBe("Intro line."); + }); + + it("finds ## and ### headings and ignores lines inside code fences", () => { + expect(sections.map((s) => s.heading)).toEqual([ + "Create a token", + "Scope presets", + "Revoke a token", + ]); + }); + + it("uses an explicit [#id] and falls back to the generated slug", () => { + expect(sections.map((s) => s.id)).toEqual([ + "create-a-token", + "scope-presets", + "revoke-a-token", + ]); + }); + + it("includes subsections in a section but not in its search text", () => { + const create = sections[0]; + expect(create?.markdown).toContain("### Scope presets"); + expect(create?.markdown).toContain("## also not a heading"); + expect(create?.markdown).not.toContain("## Revoke a token"); + expect(create?.text).toBe("Open Settings."); + }); + + it("ends a subsection at the next heading of the same or higher level", () => { + const presets = sections[1]; + expect(presets?.markdown).toContain("Pick a preset."); + expect(presets?.markdown).not.toContain("Click delete."); + }); + + it("numbers repeated headings the way the docs site does", () => { + const { sections: repeated } = splitSections( + "## Install\n\none\n\n## Install\n\ntwo\n\n## Install\n\nthree" + ); + expect(repeated.map((s) => s.id)).toEqual([ + "install", + "install-1", + "install-2", + ]); + }); + + it("returns no sections for a page without headings", () => { + const result = splitSections("# Title\n\nJust text."); + expect(result.sections).toEqual([]); + expect(result.intro).toBe("Just text."); + }); +}); + +describe("slugifyHeading", () => { + it("lowercases, drops punctuation, and joins words with hyphens", () => { + expect(slugifyHeading("Who can sign in?")).toBe("who-can-sign-in"); + expect(slugifyHeading("`data-hub-watcher service install`")).toBe( + "data-hub-watcher-service-install" + ); + expect(slugifyHeading("1. Create an access token")).toBe( + "1-create-an-access-token" + ); + }); +}); + +describe("toPlainText", () => { + it("keeps link text and drops the URL", () => { + expect(toPlainText("See [Manage tokens](/docs/manage-tokens).")).toBe( + "See Manage tokens." + ); + }); + + it("keeps the wording in component attributes and drops the tags", () => { + expect( + toPlainText( + '\n\n' + ) + ).toBe("Retire. Stop uploads."); + }); + + it("drops code fence lines but keeps the code", () => { + expect(toPlainText("```bash\nuv tool install data-hub-watcher\n```")).toBe( + "uv tool install data-hub-watcher" + ); + }); + + it("removes emphasis and inline code marks without leaving gaps", () => { + expect(toPlainText("Use **bold** and `code`.")).toBe("Use bold and code."); + }); + + it("keeps angle brackets that are part of the text", () => { + expect(toPlainText("DELETE /api/v1/tokens/ now")).toBe( + "DELETE /api/v1/tokens/ now" + ); + }); + + it("drops blockquote markers and table divider rows", () => { + const table = "| Status | Meaning |\n| --- | --- |\n| Online | Recent |"; + expect(toPlainText(table)).toBe("Status Meaning Online Recent"); + expect(toPlainText("> A quoted note")).toBe("A quoted note"); + }); +}); + +describe("makeExcerpt", () => { + const long = `${"filler words ".repeat(40)}the revoke button sits here ${"more filler ".repeat(40)}`; + + it("returns short text unchanged", () => { + expect(makeExcerpt("Short answer.", ["answer"])).toBe("Short answer."); + }); + + it("centers on the first matching term and marks the cut ends", () => { + const excerpt = makeExcerpt(long, ["revoke"]); + expect(excerpt).toContain("revoke"); + expect(excerpt.startsWith("…")).toBe(true); + expect(excerpt.endsWith("…")).toBe(true); + expect(excerpt.length).toBeLessThanOrEqual(310); + }); + + it("starts at the beginning when no term appears", () => { + const excerpt = makeExcerpt(long, ["nothing"]); + expect(excerpt.startsWith("filler")).toBe(true); + expect(excerpt.endsWith("…")).toBe(true); + }); +}); + +describe("parseDocRef", () => { + it("passes a plain page ID through", () => { + expect(parseDocRef("manage-tokens")).toEqual({ page: "manage-tokens" }); + }); + + it("accepts a docs path, a full URL, and a .md suffix", () => { + expect(parseDocRef("/docs/manage-tokens").page).toBe("manage-tokens"); + expect( + parseDocRef("https://datahub.example.com/docs/manage-tokens").page + ).toBe("manage-tokens"); + expect(parseDocRef("docs/manage-tokens.md").page).toBe("manage-tokens"); + }); + + it("reads a section from a URL fragment, and lets the argument win", () => { + expect( + parseDocRef("https://datahub.example.com/docs/manage-tokens#revoke") + ).toEqual({ page: "manage-tokens", section: "revoke" }); + expect(parseDocRef("manage-tokens#revoke", "create-a-token")).toEqual({ + page: "manage-tokens", + section: "create-a-token", + }); + }); + + it("leaves a changelog ID alone", () => { + expect( + parseDocRef("changelog/2026-10-08-hina-large-image-previews") + ).toEqual({ page: "changelog/2026-10-08-hina-large-image-previews" }); + }); +}); + +describe("findDoc", () => { + it("returns a docs page and a changelog entry by ID", () => { + expect(findDoc("manage-tokens")?.kind).toBe("docs"); + expect( + findDoc("changelog/2026-10-08-hina-large-image-previews")?.kind + ).toBe("changelog"); + }); + + it("returns nothing for an unknown ID", () => { + expect(findDoc("no-such-page")).toBeUndefined(); + }); + + it("gives a changelog entry the date in its heading and its docs link", () => { + const entry = findDoc("changelog/2026-10-08-hina-large-image-previews"); + expect(entry?.kind).toBe("changelog"); + if (entry?.kind !== "changelog") { + return; + } + expect(entry.markdown).toContain("(2026-10-08)"); + expect(entry.markdown).toContain("Related docs: "); + expect(entry.docsUrl).toContain("/docs/instrument-preprocessing"); + }); + + it("keeps a page's intro as plain text, apart from its sections", () => { + const entry = findDoc("manage-tokens"); + expect(entry?.introText.length).toBeGreaterThan(0); + expect(entry?.introText).not.toContain("#"); + expect(entry?.sections.every((section) => section.id !== "")).toBe(true); + }); +}); + +describe("unknownPageMessage", () => { + const closest = (page: string) => + /Closest: ([^.]+)\./.exec(unknownPageMessage(page))?.[1]?.split(", ") ?? []; + + it("suggests the page for a typo, a missing word, or another word form", () => { + expect(closest("manage-tokns")[0]).toBe("manage-tokens"); + expect(closest("manage-token")[0]).toBe("manage-tokens"); + expect(closest("install-watcher")).toContain("install-the-watcher"); + expect(closest("troubleshooting")).toContain("troubleshoot-a-watcher"); + }); + + it("suggests nothing for an unrelated name, but still lists every page", () => { + const message = unknownPageMessage("zzzz"); + expect(message).not.toContain("Closest:"); + expect(message).toContain("Docs pages: overview,"); + }); + + it("caps the suggestions at three", () => { + expect(closest("watcher").length).toBeLessThanOrEqual(3); + }); +}); diff --git a/web/tests/unit/mcp-docs-search.test.ts b/web/tests/unit/mcp-docs-search.test.ts new file mode 100644 index 00000000..e224a540 --- /dev/null +++ b/web/tests/unit/mcp-docs-search.test.ts @@ -0,0 +1,191 @@ +import { describe, expect, it } from "vitest"; +import { + type ChangelogDocEntry, + getDocsBundle, + listDocPages, +} from "@/lib/mcp/docs/bundle"; +import { searchDocs } from "@/lib/mcp/docs/search"; + +// Real questions a person might put to an assistant, each with the page or +// pages that answer it. The right page has to be among the top three, so a +// docs rewrite or a search change that buries it fails here. Add questions as +// people report searches that missed. +const GOLDEN_QUESTIONS: Array<{ question: string; pages: string[] }> = [ + { + question: "how do I create an access token for the watcher", + pages: ["manage-tokens"], + }, + { question: "how do I revoke a token", pages: ["manage-tokens"] }, + { question: "my watcher shows offline", pages: ["troubleshoot-a-watcher"] }, + { + question: "files are not being detected by the watcher", + pages: ["troubleshoot-a-watcher"], + }, + { + question: "how do I run the watcher as a Windows service", + pages: ["windows-service"], + }, + { + question: "what does the stalled run status mean", + pages: ["browse-runs", "concepts"], + }, + { question: "how do I make someone an admin", pages: ["manage-members"] }, + { question: "who is allowed to sign in", pages: ["security"] }, + { question: "connect Claude Code to Data Hub", pages: ["mcp"] }, + { + question: "how do I set up a new instrument", + pages: ["set-up-an-instrument"], + }, + { question: "retire an instrument", pages: ["retire-an-instrument"] }, + { + question: "which instruments does Data Hub process files for", + pages: ["instrument-preprocessing"], + }, + { + question: "how do I get Slack direct messages for new runs", + pages: ["configure-notifications", "integrations"], + }, + { question: "set up Linear for feedback reports", pages: ["integrations"] }, + { + question: "how does the watcher decide which files make up a run", + pages: ["configure-run-detection", "concepts"], + }, + { + question: "force a mandatory watcher update", + pages: ["watcher-releases"], + }, + { + question: "what command updates the watcher", + pages: ["install-the-watcher", "cli-reference"], + }, + { + question: "what is a heartbeat", + pages: ["concepts", "manage-watchers"], + }, + { question: "how do I deploy Data Hub myself", pages: ["self-hosting"] }, + { + question: "how do I call the REST API", + pages: ["api"], + }, + { + question: "large Hina Microscope image preview", + pages: ["changelog/2026-10-08-hina-large-image-previews"], + }, +]; + +// Questions the docs don't answer. Search still finds sections that share a +// word, so each must come back with no section counted as an answer, which is +// what makes the agent offer feedback. Remove a question once the docs cover it. +const UNCOVERED_QUESTIONS = [ + "how do I export runs to Benchling", + "does Data Hub support SAML single sign-on", + "is there a mobile app", + "can I schedule a weekly email digest", + "how do I translate the interface into French", + "does it integrate with Google Drive", +]; + +describe("docs search", () => { + it.each(GOLDEN_QUESTIONS)("finds $pages for “$question”", ({ + question, + pages, + }) => { + const { hits, goodMatches } = searchDocs(question); + const top = hits.slice(0, 3).map((hit) => hit.entry.page); + expect( + pages.some((page) => top.includes(page)), + `expected one of ${pages.join(", ")} in the top three, got ${top.join(", ")}` + ).toBe(true); + expect(goodMatches).toBeGreaterThan(0); + }); + + it.each(UNCOVERED_QUESTIONS)("counts no answer for “%s”", (question) => { + expect(searchDocs(question).goodMatches).toBe(0); + }); + + it("names the words of the question the docs never use, as typed", () => { + expect( + searchDocs("how do I export runs to Benchling").missingWords + ).toEqual(["Benchling"]); + expect(searchDocs("how do I revoke a token").missingWords).toEqual([]); + }); + + it("returns nothing for a question made only of filler words", () => { + expect(searchDocs("how do I").hits).toEqual([]); + }); + + it("returns nothing for words the docs never use", () => { + expect(searchDocs("zxqv plorgnak").hits).toEqual([]); + }); + + it("returns at most five results and two from one page", () => { + const { hits } = searchDocs("token"); + expect(hits.length).toBeLessThanOrEqual(5); + const perPage = new Map(); + for (const hit of hits) { + perPage.set(hit.entry.page, (perPage.get(hit.entry.page) ?? 0) + 1); + } + expect(Math.max(...perPage.values())).toBeLessThanOrEqual(2); + }); + + it("puts the section that matched in the excerpt", () => { + const { hits } = searchDocs("revoke a token"); + const revoke = hits.find((hit) => hit.section?.id === "revoke-a-token"); + expect(revoke?.excerpt.length).toBeGreaterThan(0); + expect(revoke?.excerpt.length).toBeLessThanOrEqual(320); + }); + + it("finds a changelog entry by its wording", () => { + const { hits } = searchDocs("Hina large image preview smaller"); + expect(hits.some((hit) => hit.entry.kind === "changelog")).toBe(true); + }); +}); + +describe("docs bundle", () => { + it("lists the hand-written pages and leaves out the generated catalogs", () => { + const ids = listDocPages().map((doc) => doc.page); + expect(ids).toContain("manage-tokens"); + expect(ids).toContain("cli-reference"); + expect(ids).toContain("mcp"); + expect(ids).toContain("api"); + expect(ids).not.toContain("mcp/tools"); + expect(ids).not.toContain("mcp/prompts"); + expect(ids).not.toContain("mcp/resources"); + expect(ids.every((id) => !id.startsWith("changelog/"))).toBe(true); + }); + + it("gives every page a title and a description", () => { + for (const doc of listDocPages()) { + expect(doc.title, doc.page).not.toBe(""); + expect(doc.description, doc.page).not.toBe(""); + } + }); + + it("includes changelog entries as separate documents", () => { + const changes = getDocsBundle().filter( + (entry): entry is ChangelogDocEntry => entry.kind === "changelog" + ); + expect(changes.length).toBeGreaterThan(0); + for (const change of changes) { + expect(change.page.startsWith("changelog/")).toBe(true); + expect(change.date).toMatch(/^\d{4}-\d{2}-\d{2}$/); + } + }); + + // Claude Code caps one tool result at 25,000 tokens. Three characters per + // token is a deliberately pessimistic estimate for Markdown with code in it. + it("keeps every page small enough to return in one tool result", () => { + const limitChars = 25_000 * 3; + for (const entry of getDocsBundle()) { + const result = JSON.stringify({ markdown: entry.markdown }); + expect(result.length, entry.page).toBeLessThan(limitChars); + } + }); + + it("gives every section on a page a different ID", () => { + for (const entry of getDocsBundle()) { + const ids = entry.sections.map((section) => section.id); + expect(new Set(ids).size, entry.page).toBe(ids.length); + } + }); +}); diff --git a/web/tests/unit/mcp-instructions.test.ts b/web/tests/unit/mcp-instructions.test.ts new file mode 100644 index 00000000..eb7fc2f5 --- /dev/null +++ b/web/tests/unit/mcp-instructions.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from "vitest"; +import { MCP_SERVER_INSTRUCTIONS } from "@/lib/mcp/instructions"; + +describe("MCP server instructions", () => { + // Claude Code keeps the first 2 KB of server instructions and drops the + // rest, so text past the limit never reaches the model. + it("fits in the 2 KB Claude Code keeps", () => { + expect(Buffer.byteLength(MCP_SERVER_INSTRUCTIONS)).toBeLessThanOrEqual( + 2048 + ); + }); + + it("sends how-to questions about Data Hub to the docs tools", () => { + expect(MCP_SERVER_INSTRUCTIONS).toContain("search_docs"); + expect(MCP_SERVER_INSTRUCTIONS).toContain("read_doc"); + }); + + it("names the feedback tools the server registers", () => { + expect(MCP_SERVER_INSTRUCTIONS).toContain("send_feedback"); + expect(MCP_SERVER_INSTRUCTIONS).toContain("add_run_comment"); + }); +});