From 06de36b8d1fe9114cd8acf23cf38996335ca4787 Mon Sep 17 00:00:00 2001 From: Wasim Amiri <7220175+wasimxyz@users.noreply.github.com> Date: Fri, 16 Oct 2026 15:47:55 -0700 Subject: [PATCH] Rewrite the agent instructions and split them by folder The root AGENTS.md now holds only the rules every task needs. Each folder with its own conventions gets a short AGENTS.md next to its code. The rules come from the last 25 pull requests and were checked against the code, and each file names the agent skills and external docs for the code it covers. Also turns off the AGENTS.md block that Next.js writes on every `next dev`, removes web/CLAUDE.md and the Cursor-only rules, moves the Cursor Cloud notes to docs/developer/cursor-cloud.md, and fixes stale developer docs. Co-authored-by: Cursor --- .cursor/rules/code-comments.mdc | 33 ------ .cursor/rules/run-checks.mdc | 7 -- AGENTS.md | 94 +++++++---------- README.md | 2 +- docs/developer/README.md | 1 + docs/developer/ci-and-deployment.md | 14 ++- docs/developer/cursor-cloud.md | 64 ++++++++++++ docs/developer/getting-started.md | 2 +- docs/developer/local-development.md | 10 +- docs/developer/mcp-apps.md | 2 +- docs/developer/testing.md | 4 +- docs/site/AGENTS.md | 107 ++++---------------- docs/site/next.config.mjs | 3 + infra/AGENTS.md | 23 +++++ lambda/AGENTS.md | 16 +++ packages/shared/AGENTS.md | 7 ++ watcher/AGENTS.md | 28 ++--- web/AGENTS.md | 44 +++++++- web/CLAUDE.md | 1 - web/components/AGENTS.md | 20 ++++ web/lib/api/AGENTS.md | 6 ++ web/lib/db/AGENTS.md | 9 ++ web/lib/mcp/AGENTS.md | 13 +++ web/mcp-apps/AGENTS.md | 9 ++ web/next.config.mjs | 4 + web/scripts/docs-screenshots/environment.ts | 2 +- web/tests/AGENTS.md | 6 ++ 27 files changed, 317 insertions(+), 214 deletions(-) delete mode 100644 .cursor/rules/code-comments.mdc delete mode 100644 .cursor/rules/run-checks.mdc create mode 100644 docs/developer/cursor-cloud.md create mode 100644 infra/AGENTS.md create mode 100644 lambda/AGENTS.md create mode 100644 packages/shared/AGENTS.md delete mode 100644 web/CLAUDE.md create mode 100644 web/components/AGENTS.md create mode 100644 web/lib/api/AGENTS.md create mode 100644 web/lib/db/AGENTS.md create mode 100644 web/lib/mcp/AGENTS.md create mode 100644 web/mcp-apps/AGENTS.md create mode 100644 web/tests/AGENTS.md diff --git a/.cursor/rules/code-comments.mdc b/.cursor/rules/code-comments.mdc deleted file mode 100644 index fabc2ea6..00000000 --- a/.cursor/rules/code-comments.mdc +++ /dev/null @@ -1,33 +0,0 @@ ---- -description: Code comment style and when to comment -globs: **/*.{ts,tsx,mjs,js,py} -alwaysApply: true ---- - -# Code comments - -## Style - -- Explain **why**, not **what**. The code already shows what; comments justify a decision, flag a constraint, or warn about a non-obvious interaction. -- Plain prose, sentence case, periods. No banner art or decorative dashes. -- Reference symbols and paths in backticks (`` `useChat` ``, `lib/db/...`). -- Active voice ("Returns null on..." not "Null is returned on..."). -- Reference constants by name; don't restate their value. - -## Comment when - -A future reader would otherwise have to spelunk to answer "why does it do this?": - -- Architectural choices and invariants the type system can't express. -- Workarounds for bugs / browser quirks / library limits — link the issue and note when it can be removed. -- Non-obvious tradeoffs ("we re-render on every keystroke because the sidebar live-mirrors the value"). - -## Don't comment - -- Restating the line below (`// increment the counter`). -- Standard library / framework behavior (`autoFocus = true` doesn't need a paragraph explaining what `autoFocus` does). -- TODOs without an owner or ticket — open an issue instead. - -## Length - -Inline comments should be **3 lines or fewer**. File headers cap at **4–5 lines**. Go longer only when a reader would genuinely be lost without it; otherwise trim. If a long comment only restates what the code does, delete it. diff --git a/.cursor/rules/run-checks.mdc b/.cursor/rules/run-checks.mdc deleted file mode 100644 index 07a14ec0..00000000 --- a/.cursor/rules/run-checks.mdc +++ /dev/null @@ -1,7 +0,0 @@ ---- -description: Ensure that all code changes are checked for formatting, lint, and type errors by running `make check` after every update. -globs: **/*.{ts,tsx,mjs,js,py} -alwaysApply: true ---- - -Run formatting, lint, and type checks using `make check` after ALL changes have been made. diff --git a/AGENTS.md b/AGENTS.md index b304b8f0..73291b99 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,79 +1,63 @@ # AGENTS.md -## Commits and pull requests +Data Hub collects files from lab instrument PCs (`watcher/`), processes them in AWS Lambda (`lambda/`), and serves them through a Next.js web app, REST API, and MCP server (`web/`). Developer docs start at `docs/developer/README.md`. -Do **not** put sensitive information from internal environments (production, staging, or a coworker's machine) into commit messages, PR titles/bodies, review comments, or committed files. That includes: +## Commits and pull requests -- Credentials and secrets: PATs (`dhub_…`), `AUTH_SECRET`, AWS keys/session tokens, database passwords, webhook secrets -- Pre-signed or authenticated URLs (S3 download links, anything with `X-Amz-Security-Token` / signature query params) -- Identifiers from internal environments: instrument IDs, run IDs, file IDs, and real filenames from those runs -- Raw dumps from internal MCP tools, API responses, or logs that embed the above -- `.env` contents or other gitignored config copied into the diff +Keep sensitive information from internal environments (production, staging, or a coworker's machine) out of commit messages, pull request titles and bodies, review comments, and committed files. That includes: -Safe to reference: public docs URLs, redacted error messages, and synthetic fixtures. When a PR needs to describe a production incident, summarize the failure mode — don't paste tokens, signed URLs, IDs, or full internal payloads. +- Credentials and secrets: personal access tokens (`dhub_…`), `AUTH_SECRET`, AWS keys and session tokens, database passwords, and webhook secrets. +- Pre-signed or authenticated URLs, such as S3 download links or anything with `X-Amz-Security-Token` or signature query parameters. +- Identifiers from internal environments: instrument IDs, run IDs, file IDs, and real filenames from those runs. +- Raw output from internal MCP tools, API responses, or logs that contains any of the above. +- `.env` contents or other gitignored config. -## Documentation +Public docs URLs, redacted error messages, and synthetic fixtures are safe. To describe a production incident, summarize the failure mode. -Documentation lives in `docs/`, split by who is reading: +## Changelog -- `docs/site/` is the public docs site (Fumadocs on Next.js) at https://datahub.arcadiascience.com/docs. It covers installing a watcher, setting up an instrument, managing tokens, security and permissions, and the API, MCP, and watcher CLI reference. Pages are MDX files in `docs/site/content/docs/`. Read `docs/site/AGENTS.md` before editing it. For "how do I use Data Hub" questions, search the published site first and don't rely on training data or guess at UI flows: its `/docs/llms.txt` and `/docs/llms-full.txt` routes (and a `.md` suffix on any page URL) serve clean Markdown that's cheap to fetch. -- `docs/developer/` covers contributing to and self-hosting Data Hub itself: architecture internals, local dev setup (`getting-started.md`, `local-development.md`), conventions, the step-by-step self-hosting guide for the web app and AWS infrastructure (`first-time-deployment.md`) plus CI/ongoing-deploy reference (`ci-and-deployment.md`), and per-package references (`lambda.md`, `watcher.md`, `shared-library.md`). These are plain Markdown read on GitHub. The docs site does not render them. See `docs/developer/README.md` for the full index. +- A user-visible change to the web app, file processing, the API, or AI assistant behavior needs an entry in `web/content/changelog/` in the same branch. +- Name the file `YYYY-MM-DD-short-slug.md` with today's date, and rename it if the pull request merges into `staging` on a later day. The tests for `web/lib/changelog/parse.ts` check the name and frontmatter. +- If the change needs a new watcher version, the entry names that version. +- Tests, docs, CI, dependency updates, infrastructure, and watcher-only releases don't get entries. -The docs site deploys from the `production` branch, so a docs change goes live when `staging` is promoted, together with the feature it describes. Change the docs in the same branch as the feature. +## Generated snapshots -### Watcher version +- After you change the REST API, MCP tools, prompts, or resources, the watcher CLI or version, a page in `docs/site/content/docs/`, or a dependency in a `package.json` or lockfile, run `make docs-catalogs` and commit every `*.snapshot.json` it changes. Dependency upgrades count because they can change the generated schemas. +- Never edit a `*.snapshot.json` file by hand. CI regenerates each one and fails when the committed copy differs. -Shipped watcher changes (`watcher/src/`) need a version bump in the same branch. Follow `watcher/AGENTS.md`: bump `[project].version` in `watcher/pyproject.toml` once if it still matches the merge-base, then run `uv lock`, then run `make py-watcher-cli-catalog docs-bundle-generate` and commit the updated `docs/site/src/lib/cli-catalog.snapshot.json`, which records the version, and `web/lib/mcp/docs/docs-bundle.snapshot.json`, which embeds the CLI reference page. Do not tag the release from the feature branch. +## Docs -### Changelog +- The docs site deploys from `production`, so change the docs in the same branch as the feature. +- To learn how Data Hub behaves for users, read `docs/site/content/docs/` first. The published site at https://datahub.arcadiascience.com/docs shows what is live on `production`. -User-visible changes need a dated entry in `web/content/changelog/` in the same branch. That includes the web app, file processing, the API, and AI assistant behavior. +## Testing -Name the file `YYYY-MM-DD-short-slug.md`. The date is the day the pull request merges into `staging`. The slug is lowercase words separated by hyphens. The file starts with YAML frontmatter (`title`, and `docs` when a docs page explains the change) and a short markdown body. If the change needs a new watcher version, the entry names that version. Watcher versions do not get their own entries. +Before you say a change is done: -Do not add an entry for tests, docs, CI, dependency updates, or infrastructure. +- Run `make check`. It rewrites files to fix formatting, so review the diff afterward. +- Run the unit tests for each package you touched: `make fe-test-unit` for `web/` and `make py-test-unit` for the Python packages. +- If the change touches API routes, the database schema, or the watcher-to-server protocol, also run `make fe-test-integration` or `make py-test-integration`. Both run `next build` into `web/.next`, so stop `make dev` first. +- Check UI changes in the running app. `web/AGENTS.md` says how. +- Make sure a new test fails without your change by breaking the code once. +- CI runs everything else. -### Reference snapshots (docs site) +## Before you push -The API, MCP, and [Watcher CLI](https://datahub.arcadiascience.com/docs/cli-reference) pages on the docs site render from three JSON files committed in `docs/site/src/lib/`: `openapi.snapshot.json`, `mcp-catalog.snapshot.json`, and `cli-catalog.snapshot.json`. They are generated from the code and never edited by hand. +- Once the code is final, reread every comment, docs line, and changelog sentence you wrote and check it against the code. +- After you rename or move a file, search for the old path in imports, the `Makefile`, `package.json` scripts, `.github/workflows/`, and the docs. -1. Change the REST API (`web/lib/api/openapi/`), the MCP tools, prompts, or resources (`web/lib/mcp/`), or the watcher CLI help or options (`watcher/src/data_hub_watcher/cli.py`). -2. Run `make docs-catalogs`. To refresh only the watcher CLI file, run `make py-watcher-cli-catalog docs-bundle-generate`, because the [docs bundle for the MCP server](#docs-bundle-for-the-mcp-server) embeds the CLI reference page. -3. Commit the changed snapshot files in the same branch. +## Python -CI regenerates each file and fails when the committed copy differs. +Shipped Python in `lambda/`, `watcher/`, and `packages/shared/` must run on Python 3.12, because the Lambda image and lab PCs use 3.12. Local development uses 3.13 from `.python-version`. Run Python tools through `uv run`. -### Docs bundle for the MCP server +## Skills -The MCP server answers questions about Data Hub from a bundled copy of the docs pages, `web/lib/mcp/docs/docs-bundle.snapshot.json`. It is generated from `docs/site/content/docs/` (and the watcher CLI snapshot) and never edited by hand. After changing a docs page or the watcher CLI snapshot, including a watcher version bump, run `make docs-catalogs` and commit the result in the same branch. CI regenerates it and fails when the committed copy differs. See `docs/developer/mcp-docs.md`. +- When an `AGENTS.md` names a skill, use it if it is installed. Otherwise load it for the session with `npx skills use --skill `, or with `npx skills use ` when the `AGENTS.md` gives a pinned URL. +- Ask before you install a skill that no `AGENTS.md` names, and never pass `-y`. +- When a skill's advice conflicts with an `AGENTS.md` rule, follow the `AGENTS.md`. +- `ultracite` (haydenbleasel/ultracite): fixing lint and format errors in `web/` and `docs/site/`. ## Cursor Cloud specific instructions -Data Hub is a multi-component repo (see `README.md`). The component you can run end-to-end locally with zero external credentials is the **Next.js web app + REST API + PostgreSQL** (`web/`). The `lambda/`, `watcher/`, and `packages/shared/` Python packages are exercised via tests and a local S3 mirror — no real AWS is needed for local work. - -Standard commands live in the `Makefile`, `web/package.json`, `docs/developer/getting-started.md`, and `docs/developer/local-development.md`. The notes below are the non-obvious caveats that those docs don't make obvious for a fresh cloud VM (where the update script has already installed deps). - -### Starting services (not handled by the update script) - -- **PostgreSQL must be started on every fresh VM** — the cluster is installed and the data (roles + databases) persist in the snapshot, but the server process is not running at boot: `sudo pg_ctlcluster 16 main start` (or `sudo service postgresql start`). -- Postgres is reachable at `postgres://postgres:postgres@127.0.0.1:5432`. Databases `data-hub-local` (dev) and `data_hub_test` (integration tests) already exist. The integration harness (`web/tests/integration/global-setup.ts`) hardcodes these same credentials and creates `data_hub_test` itself if missing. -- **Web dev server:** `make dev` (Next.js + Turbopack on http://localhost:3000). Sign in at `/login` with the "Sign in (dev)" button using email `alice@example.com` (workspace admin; shared seed password is submitted invisibly). - -### Environment file - -`web/.env` is gitignored and required for `make dev` / seeding. If it is missing on a fresh VM, recreate it from the "Minimal `.env`" block in `docs/developer/local-development.md` (the key lines are `DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/data-hub-local`, a 32+ char `AUTH_SECRET`, `BETTER_AUTH_URL=http://localhost:3000`, dummy `AWS_*` values, and `LOCAL_S3_MIRROR=../lambda/.local-s3`). - -### Node / Python toolchain - -- Use **Node 24 (npm 11)** — it is the nvm default and is what CI uses. `npm ci` against the committed `web/package-lock.json` **fails under npm 10** ("Missing: esbuild@… from lock file"), so don't downgrade. A clean login shell already selects Node 24 via nvm. -- Python is managed by `uv` (Python 3.13, pinned in `.python-version`). Run Python tools through `uv run …` (e.g. `uv run pytest`); the Makefile targets already do this. - -### Seeding and local file bytes - -- `make db-reseed` resets + pushes the Drizzle schema + seeds deterministic data. It prints a personal access token (`dhub_…`) for the dev user — use it for `Authorization: Bearer` API calls. -- The seed's fixture-processing step is **skipped if the dev server isn't running** (it prints a hint). To populate processed artifacts (gel-doc PNGs, plate-reader CSVs, qPCR metadata), start `make dev` first, then run `npm run db:process-fixtures` from `web/`. - -### Testing caveat - -- `make fe-test-integration` and `make py-test-integration` run `next build` + `next start`, which writes to `web/.next` and **contends with a running `make dev`** (also using `.next`). Stop the dev server before running integration tests, then restart it afterward. -- Lint/format/typecheck: `make check` (note: `py-format`/`fe-format`/`docs-format` auto-rewrite files; use `uv run ruff check .`, `npm run lint:check` (Biome formatter + linter, read-only), and `npm run typecheck` for read-only checks). `make check` also covers the docs site, which has its own dependencies: run `npm ci` in `docs/site/` first. `make docs-links` checks the links between the docs, the web app, and the changelog. +In a Cursor Cloud VM, read `docs/developer/cursor-cloud.md` before you run anything. Postgres may be missing or stopped, and `web/.env` may be missing. diff --git a/README.md b/README.md index 077182cb..7b69e06d 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ Docs are split by who is reading: ## Run it locally -The web app, API, and database run with no AWS or Google credentials, which is enough for work on the UI, REST endpoints, MCP tools, or the schema. You need PostgreSQL 15 or later and Node.js 22 or later: +The web app, API, and database run with no AWS or Google credentials, which is enough for work on the UI, REST endpoints, MCP tools, or the schema. You need PostgreSQL 15 or later and Node.js 24 with npm 11: ```sh cd web && npm install && cd .. diff --git a/docs/developer/README.md b/docs/developer/README.md index c8e72d3b..0cda4cae 100644 --- a/docs/developer/README.md +++ b/docs/developer/README.md @@ -11,6 +11,7 @@ the docs site does not render them. - [Getting started](getting-started.md) — development setup, environment variables, running locally - [Local development](local-development.md) — zero-credential dev workflow for the web app + API + database (no watcher / Lambda needed) +- [Cursor Cloud](cursor-cloud.md) — PostgreSQL, `web/.env`, and toolchain setup on a Cursor Cloud VM, plus the recommended dashboard install script - [Architecture](architecture.md) — system overview, data flow, and design decisions - [MCP Apps](mcp-apps.md) — run-report View: host render flow, the three app-only tools, the shared data source, `mcp-apps:build`, and the gitignored HTML artifact - [Docs and changelog over MCP](mcp-docs.md) — how the `search_docs`, `read_doc`, and `search_changelog` tools get their content, how search works, how changelog entry links work, and what to update when docs change diff --git a/docs/developer/ci-and-deployment.md b/docs/developer/ci-and-deployment.md index c74e83b2..672e6afe 100644 --- a/docs/developer/ci-and-deployment.md +++ b/docs/developer/ci-and-deployment.md @@ -2,7 +2,7 @@ ## GitHub Actions -Five workflows run on pushes to `staging`/`production` and on pull requests targeting those branches. A sixth (`apply-migrations.yml`) runs on merges to `staging`/`production` that touch migration files, and a seventh (`publish-watcher.yml`) runs only on `watcher-v*` tag pushes and manual dispatch. +Six workflows run on pushes to `staging`/`production` and on pull requests targeting those branches. `apply-migrations.yml` runs on merges to `staging`/`production` that touch migration files, `deploy-lambda.yml` runs on merges that touch the Lambda or its infrastructure (see [Lambda (AWS)](#lambda-aws)), and `publish-watcher.yml` runs only on `watcher-v*` tag pushes and manual dispatch. `docs-screenshots.yml` runs monthly and on demand (see [Docs screenshots](local-development.md#docs-screenshots)). ### Python lint and typecheck (`python-lint.yml`) @@ -15,7 +15,7 @@ Five workflows run on pushes to `staging`/`production` and on pull requests targ - Starts a Postgres 17 service container. - Installs both Node.js 24 and Python packages. - `make py-test` — runs all pytest tests (unit and integration). Integration tests build and start a real Next.js production server, seed a test database, and exercise the Lambda and watcher against the live API. -- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot. The MCP docs bundle embeds the same page, so run `make py-watcher-cli-catalog docs-bundle-generate` and commit both files, or `docs-lint.yml` fails next. +- `make py-watcher-cli-catalog`, then fails if `docs/site/src/lib/cli-catalog.snapshot.json` changed. The docs site renders its watcher CLI reference from that committed file, so a change to the CLI or a watcher version bump has to commit the regenerated snapshot. The MCP docs bundle embeds the same page, so run `make docs-catalogs` and commit both files, or `docs-lint.yml` fails next. ### TypeScript lint and typecheck (`typescript-lint.yml`) @@ -40,6 +40,13 @@ Runs when `docs/**`, `web/lib/docs.ts`, `web/lib/mcp/docs/docs-bundle.snapshot.j - `make fe-test-unit` — runs `tests/unit/` and `tests/mcp/` (in-memory MCP protocol tests) together; no Postgres, no Next.js server, no global setup. See [Testing](testing.md). - `make fe-test-integration` — runs Vitest integration tests that test the API routes and MCP server over HTTP against a real database. +### Infra template validation (`infra-validate.yml`) + +Runs when `infra/**`, `deploy-lambda.yml`, `.github/scripts/check_sam_deploy_overrides.py`, or the `Makefile` changes: + +1. `make sam-validate` — lints both CloudFormation templates with cfn-lint. +2. `make sam-check-overrides` — runs the CI and Makefile `sam deploy` commands with dummy credentials and no reachable AWS endpoint, to catch parameter overrides SAM rejects before they reach AWS. + ### Apply database migrations (`apply-migrations.yml`) Triggered on pushes to `staging`/`production` (i.e. PR merges) that change files under `web/drizzle/`. The single job sets `environment: ${{ github.ref_name }}` so GitHub selects that environment's secrets and protection rules, then runs `npm run db:migrate` (Drizzle) against the environment's PostgreSQL database using the environment's `DATABASE_URL` secret. A per-branch `concurrency` group prevents overlapping migration runs. @@ -238,4 +245,7 @@ This is equivalent to: ```sh make py-check # py-format + py-lint + py-typecheck make fe-check # fe-format + fe-lint + fe-typecheck +make docs-check # docs-format + docs-lint + docs-typecheck + docs-links ``` + +The `*-format` steps rewrite files, so review the diff afterward. The docs checks need `npm ci` in `docs/site/` first. diff --git a/docs/developer/cursor-cloud.md b/docs/developer/cursor-cloud.md new file mode 100644 index 00000000..f434e7c4 --- /dev/null +++ b/docs/developer/cursor-cloud.md @@ -0,0 +1,64 @@ +# Cursor Cloud + +Notes for Cursor Cloud agents and for whoever maintains the Cloud environment. A Cloud VM can run the web app, the REST API, and PostgreSQL end to end with no external credentials. The Python packages run through their tests and the local S3 mirror, so no AWS access is needed. For the general local workflow, see [Local development](local-development.md). + +## PostgreSQL + +The tests and the seed scripts expect PostgreSQL at `127.0.0.1:5432`, with a `postgres` user whose password is `postgres`. They use two databases: `data-hub-local` for `make dev`, and `data_hub_test` for the integration tests, which create it themselves if it's missing. + +If PostgreSQL is installed but not running, which is the usual state right after a VM boots, start it: + +```sh +sudo service postgresql start +``` + +If PostgreSQL isn't installed, install it, start it, then set the password and create the dev database: + +```sh +sudo apt-get update +sudo apt-get install -y postgresql +sudo service postgresql start +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" +sudo -u postgres createdb data-hub-local +``` + +`pg_isready -h 127.0.0.1` reports whether it's accepting connections. + +## Environment file + +`web/.env` is gitignored, and `make dev` and the seed scripts need it. If it's missing, create it from the "Minimal `.env`" block in [Local development](local-development.md#minimal-env). + +## Node.js and Python + +- Use Node.js 24 with npm 11. nvm selects it by default in a login shell. `npm ci` against the committed lockfile fails under npm 10, so don't downgrade. +- Python is managed by `uv`, with the version pinned in `.python-version`. If `uv` is missing, install it with `curl -LsSf https://astral.sh/uv/install.sh | sh`, then run `uv sync --all-packages`. +- `make check` covers the docs site, which has its own dependencies. Run `npm ci` in `docs/site/` before the first `make check`. + +## Seeding, signing in, and tests + +- `make db-reseed` resets the database, pushes the schema, and seeds sample data. It prints a personal access token for the dev user. It skips fixture processing unless `make dev` is running; [Local development](local-development.md) explains how to run that step afterward. +- Sign in at `/login` with "Sign in (dev)" as `alice@example.com`. +- The integration tests build into `web/.next`, which a running `make dev` also uses, so stop the dev server before you run them. See [Testing](testing.md). + +## Environment setup in the Cursor dashboard + +Agents have had to install PostgreSQL and `uv` by hand on some VMs. Putting both in the environment's install script removes that step. Cursor runs the install script when it builds the environment and keeps the resulting disk, but not running processes, so PostgreSQL also needs a start command. + +Install script: + +```sh +command -v psql >/dev/null || (sudo apt-get update && sudo apt-get install -y postgresql) +command -v uv >/dev/null || (curl -LsSf https://astral.sh/uv/install.sh | sh) +sudo service postgresql start +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" +sudo -u postgres createdb data-hub-local 2>/dev/null || true +(cd web && npm ci) && (cd docs/site && npm ci) && uv sync --all-packages +``` + +Start command: + +```sh +sudo service postgresql start +``` + +See [Cursor's Cloud Agent setup docs](https://cursor.com/docs/cloud-agent/setup) for where these settings live. diff --git a/docs/developer/getting-started.md b/docs/developer/getting-started.md index e6b5781f..b78ff37c 100644 --- a/docs/developer/getting-started.md +++ b/docs/developer/getting-started.md @@ -8,7 +8,7 @@ This guide walks through setting up the full Data Hub development environment. | --- | --- | --- | | Python | >= 3.12 | Lambda, watcher, shared library | | [uv](https://docs.astral.sh/uv/) | latest | Python package manager and workspace orchestrator | -| Node.js | >= 22 | Web application | +| Node.js | 24, with npm 11 | Web application and docs site. `npm ci` against the committed lockfile fails under npm 10. | | PostgreSQL | >= 15 | Database (local development) | | Docker | latest | Lambda container builds | | ffmpeg | latest | DishCam TIFF → MP4 encode (`brew install ffmpeg` / `apt install ffmpeg`). Encode tests skip if it is missing. | diff --git a/docs/developer/local-development.md b/docs/developer/local-development.md index 0d61c1ce..691a59d9 100644 --- a/docs/developer/local-development.md +++ b/docs/developer/local-development.md @@ -24,8 +24,8 @@ Sign in at `/login` using the "Sign in (dev)" button with the seeded `alice@exam ## Prerequisites -- **PostgreSQL >= 15** running locally on `127.0.0.1:5432` and reachable as the OS user. -- **Node.js >= 22**. +- **PostgreSQL >= 15** running locally on `127.0.0.1:5432`, with a `postgres` user whose password is `postgres`. The reset and seed scripts refuse any other `DATABASE_URL` (see [web/scripts/README.md](../../web/scripts/README.md)). +- **Node.js 24** with npm 11. `npm ci` against the committed lockfile fails under npm 10. Python, AWS CLI, Docker, and SAM are not required for this workflow. @@ -34,7 +34,7 @@ Python, AWS CLI, Docker, and SAM are not required for this workflow. Create `web/.env` with the following. The first two are mandatory; the rest are dummy values that let signed-URL generation and IAM-flagged code paths run without hitting AWS. ```sh -DATABASE_URL=postgres://localhost:5432/data-hub-local +DATABASE_URL=postgres://postgres:postgres@127.0.0.1:5432/data-hub-local # Any 32+ character string. Better Auth uses it to sign cookies / encrypt # session data (also accepted as `BETTER_AUTH_SECRET`). @@ -223,7 +223,7 @@ Components don't need to change — every existing run viewer already fetches `/ A few details worth knowing: -- Adding fixtures for more instruments means an `INSTRUMENT_FIXTURES` entry in [web/lib/db/seed.ts](../../web/lib/db/seed.ts) keyed by the kebab-case instrument id from `data_hub_shared.enums.Instrument` (`{ files: [{ filename, contentType }, …], runIds }`) pointing at files under `lambda/tests/fixtures/`. List every fixture you want cycled across seeded runs. The handler rejects unknown instrument ids because `parse_s3_event` only accepts values from that enum. +- Adding fixtures for more instruments means an `INSTRUMENT_FIXTURES` entry in [web/lib/db/seed.ts](../../web/lib/db/seed.ts) keyed by the kebab-case instrument id of a seeded instrument (`{ files: [{ filename, contentType }, …], runIds }`) pointing at files under `lambda/tests/fixtures/`. List every fixture you want cycled across seeded runs. The handler looks up the instrument and picks a processor by its `instrument_type`, so the seeded instrument needs a type that has a processor in `lambda/src/data_hub_lambda/processors.py`. Files for other types are skipped. - The route is gated on `NODE_ENV !== "production"` AND `LOCAL_S3_MIRROR` set; either condition unmet returns 404 unconditionally, so a production build can never expose the filesystem. - The MCP tool at `/mcp/v1` returns a relative `/api/local-s3/...` URL when the mirror is active — browsers resolve it against the current origin, but non-browser MCP clients on localhost may need to prefix with `http://localhost:3000`. @@ -247,7 +247,7 @@ cd web && npm run docs:screenshots -- --serve # seed and start the app, then w How it works: - It uses its own database, `data_hub_screenshots`, and its own file mirror under `web/.docs-screenshots/`, so your local data is untouched and `make dev` can keep running. -- It runs the app with `next dev`, because the local file mirror that serves the gel images, PDFs, and plate maps is turned off in production builds. It restores `web/tsconfig.json`, `web/AGENTS.md`, and `web/next-env.d.ts`, which `next dev` edits when it starts. +- It runs the app with `next dev`, because the local file mirror that serves the gel images, PDFs, and plate maps is turned off in production builds. It restores `web/tsconfig.json` and `web/next-env.d.ts`, which `next dev` edits when it starts. - The data is the same as `make db-reseed` ([web/scripts/seed-workspace.ts](../../web/scripts/seed-workspace.ts)), plus a few things only the screenshots need ([web/scripts/docs-screenshots/setup.ts](../../web/scripts/docs-screenshots/setup.ts)): realistic access tokens, a fake Linear workspace from the feedback package's test server, sample feedback reports, and a sample AI client for the sign-in approval screen. Shots that change data, such as the manual-upload run and the deleted run, undo their change when they finish. - It signs in by writing a session row for Alice, so the dev-only sign-in form isn't needed. - Dates are real, and every run rewrites every image. Before each shot it moves the seeded watcher heartbeats forward, because a watcher shows as offline after five minutes without one. diff --git a/docs/developer/mcp-apps.md b/docs/developer/mcp-apps.md index 672e4f9c..fa74264b 100644 --- a/docs/developer/mcp-apps.md +++ b/docs/developer/mcp-apps.md @@ -2,7 +2,7 @@ Data Hub's MCP server serves the run report as an interactive HTML page, so a chat client that supports the MCP Apps extension renders the same report the website renders instead of rebuilding it from JSON. -MCP Apps ([SEP-1865](https://modelcontextprotocol.io/specification/2026-01-26/apps)) is an extension to the Model Context Protocol. The server publishes an HTML page at a `ui://` address, the client fetches that page and runs it inside a locked-down iframe, and the page asks the client to make tool calls on its behalf. Throughout this page, "host" means the chat client doing that work, and "View" means the HTML page. +MCP Apps ([SEP-1865](https://modelcontextprotocol.io/seps/1865-mcp-apps-interactive-user-interfaces-for-mcp)) is an extension to the Model Context Protocol. The server publishes an HTML page at a `ui://` address, the client fetches that page and runs it inside a locked-down iframe, and the page asks the client to make tool calls on its behalf. Throughout this page, "host" means the chat client doing that work, and "View" means the HTML page. The same React components render both surfaces. The web app feeds them over REST and the View feeds them over MCP tool calls. diff --git a/docs/developer/testing.md b/docs/developer/testing.md index 0623503b..b4401711 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -27,8 +27,8 @@ and start a real Next.js **production** server (`next build && next start`) rather than `next dev` — dev mode recompiles on every request, which makes a large suite 5–10x slower, and a production build matches actual deployment behavior. That build writes to `web/.next`, which **contends with a running -`make dev`** (same directory) — stop the dev server first. See `AGENTS.md` for -this and other local-environment caveats. +`make dev`** (same directory) — stop the dev server first. See +[Local development](local-development.md) for the rest of the local setup. ## Python (pytest): `lambda/`, `watcher/` diff --git a/docs/site/AGENTS.md b/docs/site/AGENTS.md index 37ef8094..96dbfbfd 100644 --- a/docs/site/AGENTS.md +++ b/docs/site/AGENTS.md @@ -1,97 +1,28 @@ -# AGENTS.md +# Docs site -## What this folder is +This is the public docs site at https://datahub.arcadiascience.com/docs, a Fumadocs app on Next.js 16. Next.js 16 changed APIs that most training data predates, so read the guide in `docs/site/node_modules/next/dist/docs/` before you use a Next.js API you're unsure about. Run commands from `docs/site/`, or use the `make docs-*` targets from the repo root. -This is the **public documentation site** for [Data Hub](https://github.com/Arcadia-Science/data-hub) — a standalone Fumadocs/Next.js app that lives in `docs/site/` of the `data-hub` repository. It is deployed as its own Vercel project but served under the product's domain at https://datahub.arcadiascience.com/docs (see [Microfrontends](#microfrontends) below). It holds all user-, operator-, and admin-facing documentation: installing a watcher, adding an instrument, managing tokens, security/permissions, and full CLI/config/API reference. +## Pages -If you're looking for how to *contribute to or self-host Data Hub itself* (architecture internals, local dev setup, conventions, CI/ops, deploying the web app and AWS infrastructure, package references), that content lives in `docs/developer/` next to this folder. Those are plain Markdown files read on GitHub. This site does not render them, because the site only reads `content/docs/`. Link to them from pages here with `https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/.md`; `npm run check:links` fails on a broken link or a link to another branch. +- Pages live in `docs/site/content/docs/**/*.mdx`. Each folder's `meta.json` sets the page order, so edit it when you add or remove a page. +- Every page needs `title` and `description` in its frontmatter, and starts with an `` tag naming who it is for. +- Use `` or ``, `` with ``, and `` with `` for warnings and walkthroughs. +- Link to other pages as `/docs/#`. Give a heading a stable `[#id]` when other pages link to it, because the MCP server uses those IDs as section names. +- Link to developer docs as `https://github.com/Arcadia-Science/data-hub/blob/production/docs/developer/.md`. The link check fails on other branches and on broken links. +- Each page must make sense read alone as Markdown, because agents read pages through `/llms.txt`, `.md` URLs, and the MCP server. +- A test in `web/tests/unit/mcp-docs-search.test.ts` caps page length, so split a page that grows too long. When you rename or split a page, update that test's question lists. -Run every command below from `docs/site/`, or use the `make docs-*` targets from the repository root. +## Generated pages -## Content structure +- `docs/site/content/docs/meta.json` lists `api/index` and then `...api`. Don't also list bare `api`, or the folder nests twice. Endpoint tables belong in the generated `/docs/api/*` pages, not in `docs/site/content/docs/api/index.mdx`. +- The flag tables in the CLI reference come from the CLI snapshot. Keep only narrative caveats in `docs/site/content/docs/cli-reference.mdx`. +- This package is `"type": "module"`, because `docs/site/scripts/generate-docs-bundle.ts` loads pages through `fumadocs-mdx/node`. -- Pages live in `content/docs/*.mdx`; nav order comes from `content/docs/meta.json` (edit both when adding or removing a page). -- Every page's frontmatter needs `title` and `description` — they drive page metadata and the `/llms.txt` index. -- Start each page with an `...` tag naming who it's for (lab operators, admins, engineers, developers/integrators) — pages mix audiences otherwise and readers can't tell if a page applies to them. -- Custom MDX components available: ``, ``, ``/``, ``/``. Prefer these over plain prose for multi-step walkthroughs and warnings, to match the rest of the site. -- Cross-reference other pages with relative site paths (`/docs/` or `/docs/#`), not raw `content/docs/*.mdx` file paths. +## Links and microfrontends -## OpenAPI API docs +- `make docs-links` doesn't check `#anchors`. Run it after you rename a page, because `web/lib/docs.ts` and the `docs:` links in changelog entries point at page names. +- Don't add a copy of `microfrontends.json` here. `docs/site/next.config.mjs` points `VC_MICROFRONTENDS_CONFIG` at `web/microfrontends.json`, and a missing config throws `MicrofrontendsError` in typecheck, build, and dev. -- Generated operation pages under `/docs/api/*` come from the committed snapshot `src/lib/openapi.snapshot.json` via `fumadocs-openapi` (`src/lib/openapi.ts` + `staticSource`). The docs build needs no network. -- The snapshot is generated from the API definition in `web/lib/api/openapi/`. After changing the API, run `make docs-catalogs` from the repository root and commit the result in the same branch. CI regenerates it and fails when the committed copy differs. -- `content/docs/api/index.mdx` is the API section overview (auth, first request, pagination, errors, MCP pointer) at `/docs/api`. Endpoint tables belong in the generated `/docs/api/*` pages, not that MDX file. Root `meta.json` lists `api/index` then `...api` so the overview and OpenAPI tag groups are flattened under the API section once (do not also list bare `api`, or the folder nests and duplicates). +Fumadocs docs: https://fumadocs.dev/llms.txt -## MCP catalog docs - -- Tools / prompts / resources pages render from the committed snapshot `src/lib/mcp-catalog.snapshot.json` (`src/lib/mcp-catalog.ts`). The docs build needs no network. -- The snapshot is generated from the MCP definitions in `web/lib/mcp/`. After changing MCP tools, prompts, or resources, run `make docs-catalogs` and commit the result. CI fails when the committed copy is out of date. - -## Watcher CLI catalog - -- [`content/docs/cli-reference.mdx`](content/docs/cli-reference.mdx) renders `` from the committed snapshot at `src/lib/cli-catalog.snapshot.json`. -- Regenerate it with `make py-watcher-cli-catalog docs-bundle-generate` (or `make docs-catalogs`, which also refreshes the other two snapshots) and commit the result. CI fails when it is out of date. A watcher version bump changes it too, because the snapshot records the version. The [MCP docs bundle](#mcp-docs-bundle) embeds this page, so it changes with it. -- Keep narrative caveats (interactive prompts, Windows-only service, examples) in the MDX; do not hand-maintain flag tables. -- Markdown / LLM export expands `WatcherCliCatalog` via `getLLMText` (`src/lib/cli-catalog-markdown.ts`), same placeholder pattern as the MCP catalogs. - -The three `src/lib/*.snapshot.json` files are generated output, so Biome skips them. Never edit them by hand. - -## MCP docs bundle - -The Data Hub MCP server answers questions from a copy of these pages bundled into the web app, so each deployment answers from the docs that match its own code. - -- `scripts/generate-docs-bundle.ts` writes the copy to `web/lib/mcp/docs/docs-bundle.snapshot.json`, using the same `getLLMText` output as `/docs/.md`. It leaves out the generated API endpoint pages and the MCP tool, prompt, and resource catalogs. -- Regenerate it with `make docs-catalogs` (or `npm run docs-bundle:generate` here) after changing any page or the watcher CLI snapshot, and commit the result. CI fails when it is out of date. Never edit it by hand. -- The script loads the pages through `fumadocs-mdx/node`, which needs this package to be an ES module (`"type": "module"`), and runs under `tsx`. `scripts/stub-assets-loader.mjs` stubs image imports, which Node cannot load. -- The server returns a page in one tool result, so a page must stay under 75,000 characters (the longest today is about 18,000). A test fails above that, because Claude Code caps a tool result at 25,000 tokens. Split a page that grows past it. -- Give every heading a stable `[#id]` when other pages link to it. The server uses those IDs as section names, and agents quote them back. -- Search quality is covered by a list of real questions in `web/tests/unit/mcp-docs-search.test.ts`, plus questions the docs don't answer, which must come back flagged as weak matches. When a page is renamed or split, or a new page answers one of those questions, update the lists. - -## AI-consumption surface - -This site is intentionally public and indexable, and is itself built to be read by agents: `/llms.txt` (page index), `/llms-full.txt` (full corpus dump), and `/docs/.md` (any page as raw Markdown, via content negotiation on `Accept: text/markdown`). The Data Hub MCP server serves the same pages through its `search_docs` and `read_doc` tools (see [MCP docs bundle](#mcp-docs-bundle)). When editing docs, keep pages self-contained enough that they still make sense pulled out of the nav and read as a flat Markdown file — that's how most AI clients will actually consume them. - -## Checks - -Run before pushing (`make docs-check` from the repository root runs all three, and `make docs-format` applies the formatter first): - -```sh -npm run lint:check # Ultracite (Biome) format + lint, read-only -npm run typecheck # fumadocs-mdx codegen + next typegen + tsc --noEmit -npm run check:links # page names, changelog links, and developer-doc links -``` - -`npm run typecheck` will fail on bad MDX (frontmatter, unclosed components) even if `next dev` doesn't complain, since `fumadocs-mdx` regenerates page types from `content/docs/`. - -`npm run check:links` checks that the targets exist; it does not check `#heading` anchors. It covers: - -- The `${DOCS_URL}/` page names in `web/lib/docs.ts`. -- The `docs:` links in `web/content/changelog/*.md`. -- GitHub links in `content/docs/**/*.mdx` to files in this repository. -- Relative links inside `docs/developer/*.md`. - -Renaming or deleting a page here can break the first two, so run it after changing page file names. - -## Microfrontends - -This site is deployed as a [Vercel Microfrontends](https://vercel.com/docs/microfrontends) child app mounted at `/docs` under the product web app's domain, so `next.config.mjs` wraps the app with `withMicrofrontends`. That wrapper reads the microfrontends config at build time — in `npm run typecheck`, `next build`, `next dev`, and the Vercel deploy — and throws `MicrofrontendsError` if it can't find it. - -The config lives in one place: `web/microfrontends.json`, owned by the default app (the web app). `next.config.mjs` points the wrapper at it by setting `VC_MICROFRONTENDS_CONFIG` to that file, unless the variable is already set. The `dev` and `proxy` scripts in `package.json` use the same path. Do not add a copy of `microfrontends.json` here. Changing routing, fallbacks, or `packageName` means editing `web/microfrontends.json` only. - -Reading a file outside `docs/site/` means the Vercel project's **Include source files outside of the Root Directory in the Build Step** setting must stay on. Production routing is still owned by the web app's deployed config; the file only feeds this app's own build (asset prefix and the local dev proxy). - -## Deployment - -The docs Vercel project (`data-hub-docs`) is connected to the `data-hub` repository with Root Directory `docs/site`, and its production branch is `production`. Merging to `staging` creates a preview; promoting `staging` to `production` publishes the site, at the same moment as the web app changes it describes. - -`vercel.json` runs `scripts/vercel-ignore-build.sh`, which skips the build when nothing under `docs/site/` or `web/microfrontends.json` changed since the last deployment. The web app's project does the opposite for pushes that only change `docs/`. - - - -# This is NOT the Next.js you know - -This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. - -This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean. - - +Skill: `writing-guidelines` (vercel-labs/agent-skills), for reviewing the writing on a page. diff --git a/docs/site/next.config.mjs b/docs/site/next.config.mjs index ff70ecd6..ac75e3e3 100644 --- a/docs/site/next.config.mjs +++ b/docs/site/next.config.mjs @@ -16,6 +16,9 @@ const withMDX = createMDX(); /** @type {import('next').NextConfig} */ const config = { reactStrictMode: true, + // `docs/site/AGENTS.md` carries its own Next.js guidance. Without this, + // every `next dev` writes its managed block back into that file. + agentRules: false, }; export default withMicrofrontends(withMDX(config)); diff --git a/infra/AGENTS.md b/infra/AGENTS.md new file mode 100644 index 00000000..0b4cbc24 --- /dev/null +++ b/infra/AGENTS.md @@ -0,0 +1,23 @@ +# AWS infrastructure + +Read `docs/developer/ci-and-deployment.md` before you change a template. It explains which changes CI can deploy and which need an admin. + +- Pass parameter overrides through `make sam-deploy` or `.github/workflows/deploy-lambda.yml`, and keep secrets out of `infra/samconfig.toml`. +- Keep `AlarmEmail` quoted in the override list, because SAM rejects an empty `Key=` override. +- `ENABLE_S3_FILES` is `"true"` or `"false"`, never empty, and `infra/.env.` must match the GitHub environment variable of the same name. +- Buckets use `DeletionPolicy: Retain`, and `make sam-teardown` refuses `production`. +- A new `${{ … }}` expression in the deploy step of `.github/workflows/deploy-lambda.yml` needs a stand-in value in `WORKFLOW_VALUES` in `.github/scripts/check_sam_deploy_overrides.py`, or `make sam-check-overrides` fails. +- Bucket policies, lifecycle rules, the first CORS grant, the VPC and NAT gateway, alarms, and IAM roles or boundaries need an admin `make sam-deploy`, because the CI role can't change them. A lifecycle write replaces every rule, so copy any hand-added rule into the template first. +- Preview deployments may assume staging roles only. Never trust preview OIDC subjects on production. +- After you edit a template, run `make sam-validate` and `make sam-check-overrides`. They need the SAM CLI and aren't part of `make check`. + +## Docs + +- AWS SAM: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/llms.txt +- CloudFormation: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/llms.txt + +## Skills + +- `aws-cloudformation` (aws/agent-toolkit-for-aws): writing templates and diagnosing failed stacks. +- `aws-iam` (aws/agent-toolkit-for-aws): roles, trust policies, and bucket policies. +- `signing-in-to-aws` (aws/agent-toolkit-for-aws): getting credentials for an admin deploy. diff --git a/lambda/AGENTS.md b/lambda/AGENTS.md new file mode 100644 index 00000000..6a3f4f00 --- /dev/null +++ b/lambda/AGENTS.md @@ -0,0 +1,16 @@ +# Lambda + +Read `docs/developer/lambda.md` before you add or change a processor. + +- Register a new processor in `lambda/src/data_hub_lambda/processors.py`, and add its type to `PROCESSABLE_INSTRUMENT_TYPES` in `web/lib/instruments/processable-types.ts`. `lambda/tests/test_processors.py` checks that the two lists match. +- One `instrument_type` maps to one vendor's file format. Support for a second vendor needs a new type. +- Don't add S3 notifications for new instruments. The raw bucket already sends every upload to the handler, which picks the processor by type. +- Build the image with `make docker-build-lambda` from the repo root, because the build context is the whole repo. +- Production code never imports `data_hub_lambda_devtools`. + +AWS Lambda docs: https://docs.aws.amazon.com/lambda/latest/dg/llms.txt + +## Skills + +- `aws-sdk-python-usage` (aws/agent-toolkit-for-aws): boto3 code here and in `packages/shared/`. +- `troubleshooting-s3-files` (aws/agent-toolkit-for-aws): S3 Files mount problems when `EnableS3Files` is on. diff --git a/packages/shared/AGENTS.md b/packages/shared/AGENTS.md new file mode 100644 index 00000000..c08e0d82 --- /dev/null +++ b/packages/shared/AGENTS.md @@ -0,0 +1,7 @@ +# Shared Python library + +Read `docs/developer/shared-library.md` and `docs/developer/testing.md` before you change this package. + +- Code the watcher needs at runtime can't live here, because the watcher installs this package only for its tests. +- The token helpers in `packages/shared/src/data_hub_shared/testing.py` must produce the same tokens as `web/lib/tokens.ts`. +- The `Instrument` enum holds display names. Lambda picks a processor by `instrument_type` in `lambda/src/data_hub_lambda/processors.py`. diff --git a/watcher/AGENTS.md b/watcher/AGENTS.md index 2297236b..530c5b17 100644 --- a/watcher/AGENTS.md +++ b/watcher/AGENTS.md @@ -1,20 +1,24 @@ -# Watcher agent notes +# Watcher -The published package version is `[project].version` in `watcher/pyproject.toml`. Heartbeats and `data-hub-watcher --version` read that value from the installed package metadata. Lab PCs only pick up a new version after someone tags `watcher-vX.Y.Z` on `production` and advertises it in Settings → Watchers — see [Roll out watcher releases](https://datahub.arcadiascience.com/docs/watcher-releases). +The published version is `[project].version` in `watcher/pyproject.toml`. Heartbeats and `data-hub-watcher --version` read it from the installed package. Lab PCs pick up a new version only after someone tags `watcher-vX.Y.Z` on `production` and advertises it in Settings → Watchers, as described at https://datahub.arcadiascience.com/docs/watcher-releases. Commands, config, and the Windows service are covered in `docs/developer/watcher.md`. ## Bump the version when you change shipped code -If this branch changes anything under `watcher/src/` (runtime, CLI, upload, update), bump `watcher/pyproject.toml` **once on the branch** before you finish. +If this branch changes anything under `watcher/src/`, bump the version once on the branch before you finish: -1. Compare `[project].version` to the merge-base (`staging` unless the user named another base). +1. Compare `[project].version` with the merge-base (`staging`, unless the user named another base). 2. If it still matches the base, bump it: - - **patch** (`1.0.0` → `1.0.1`) — bug fixes and behavior changes that do not add a feature or break a contract - - **minor** (`1.0.1` → `1.1.0`) — new commands, flags, or backward-compatible features - - **major** (`1.1.0` → `2.0.0`) — breaking CLI, config, or API-protocol changes -3. Run `uv lock` from the repo root so `uv.lock` records the new workspace version. -4. Run `make py-watcher-cli-catalog docs-bundle-generate` from the repo root and commit both changed files. The CLI catalog snapshot records the version, and the MCP docs bundle embeds the CLI reference page. CI fails if either is out of date. -5. Do not bump again for later commits on the same branch. If the base already moved past your number, bump from the current base version instead. + - patch (`1.0.0` → `1.0.1`): bug fixes and behavior changes that don't add a feature or break a contract. + - minor (`1.0.1` → `1.1.0`): new commands, flags, or backward-compatible features. + - major (`1.1.0` → `2.0.0`): breaking CLI, config, or API protocol changes. +3. Run `uv lock` from the repo root so `uv.lock` records the new version. +4. Regenerate the snapshots as "Generated snapshots" in the root `AGENTS.md` describes, and commit both changed files. The CLI catalog records the version, and the MCP docs bundle embeds the CLI reference page. +5. Don't bump again for later commits on the branch. If the base has moved past your number, bump from the base's version instead. -Do **not** bump for tests-only, docs-only, or `AGENTS.md` edits. Do **not** tag `watcher-v*` from a feature branch. Do **not** change the seeded `watcher_release_config` latest version — that is the advertised fleet pin for local seed data, not this package version. +Changes to tests, docs, or `AGENTS.md` alone don't need a bump. Never tag `watcher-v*` from a feature branch. Leave the seeded `watcher_release_config` version alone. It pins the fleet version for local seed data and isn't this package's version. -The CLI catalog step in the root `AGENTS.md` still applies when you change Click help or options. +## Rules + +- Runtime code never imports `data_hub_shared`. It is a test-only dependency and isn't published to PyPI. +- Give every new config field a default, so the YAML files already on lab PCs keep loading. +- The `preview` environment never auto-updates lab PCs. Keep that check in `watcher/src/data_hub_watcher/updater.py`. diff --git a/web/AGENTS.md b/web/AGENTS.md index 643577df..d22cb74a 100644 --- a/web/AGENTS.md +++ b/web/AGENTS.md @@ -1,9 +1,43 @@ - +# Web app -# This is NOT the Next.js you know +## Next.js -This version has breaking changes — APIs, conventions, and file structure may all differ from your training data. Read the relevant guide in `node_modules/next/dist/docs/` (resolved from this file's directory; in monorepos the `next` package may not be visible from the repo root) before writing any code. Heed deprecation notices. +This app runs Next.js 16, which changed APIs that most training data predates. For example, middleware is now `web/proxy.ts`. Before you use a Next.js API you're unsure about, read its guide in `web/node_modules/next/dist/docs/`. -This block is written and re-added by `next dev` — verify at `node_modules/next/dist/server/lib/generate-agent-files.js`. Removing it from a diff only re-creates the uncommitted change; committing it with your work keeps the tree clean. +## Checking changes in the running app - +With `make dev` running, use the `next-dev-loop` skill. It reads errors, logs, and routes from `http://localhost:3000/_next/mcp` and drives the page with `agent-browser`. Sign in at `/login` with "Sign in (dev)" as `alice@example.com`. + +## Rules + +- Upgrade `better-auth` and every `@better-auth/*` package together, to the same version. Releases 1.7.0 through 1.7.7 break MCP client registration. +- Read and write `localStorage` only through `parseVersioned` and `serializeVersioned` in `web/lib/versioned-storage.ts`, and keep data written by the previous release readable. Tabs still open on the old release share the same storage. +- Instrument and run IDs can contain `%`, `#`, `?`, and spaces. Encode them once with `encodeURIComponent` when you build a URL, and decode them once when you read one. +- S3 returns 416 for a byte-range request on an empty object, and `Content-Type` can carry parameters such as `; charset=utf-8`. Handle both. +- `web/microfrontends.json` is the only routing config for both the web app and the docs site. Change routing, fallbacks, or `packageName` there. + +## External docs + +Check the current docs before you change an integration: + +- Slack: https://docs.slack.dev/llms.txt. Any page also works with `.md` on the end. +- Linear: https://linear.app/llms.txt. Data Hub reaches Linear through `@arcadiascience/app-feedback-toolkit`. +- Vercel: https://vercel.com/docs/llms.txt +- Better Auth: https://www.better-auth.com/llms.txt. Use the 1.7 docs to match the installed version. +- Amazon S3: https://docs.aws.amazon.com/AmazonS3/latest/userguide/llms.txt + +## Skills + +- `vercel-react-best-practices` (vercel-labs/agent-skills): React and Next.js components, data fetching, and performance. +- `next-best-practices` (vercel-labs/openreview): Next.js file conventions and Server Component boundaries. +- `next-upgrade` (vercel-labs/openreview): upgrading Next.js. +- `next-dev-loop` (vercel/next.js): checking a change in the running app. +- `agent-browser` (vercel-labs/agent-browser): driving a real browser for `next-dev-loop`. +- `nuqs`, pinned at https://github.com/pproenca/dot-skills/tree/3791a24754a7040d7015cc07bc240c4fbe481e20/skills/.curated/nuqs: URL query state. +- `zod`, pinned at https://github.com/pproenca/dot-skills/tree/c34c29a4b4fa079a18d9d3ee9f727aa11197ff3b/skills/.curated/zod: Zod schemas in the API, MCP tools, and forms. +- `drizzle-orm-patterns`, pinned at https://github.com/giuseppe-trisciuoglio/developer-kit/tree/08a045b2694b090d9d93019f6395c1cbbc0f1011/plugins/developer-kit-typescript/skills/drizzle-orm-patterns: Drizzle queries, transactions, and schema. +- `better-auth-best-practices`, `better-auth-security-best-practices`, and `create-auth` (better-auth/skills): auth config, hardening, and adding a sign-in method. +- `vercel-microfrontends` (vercel/microfrontends): changes to `web/microfrontends.json`. +- `aws-sdk-js-v3-usage` (aws/agent-toolkit-for-aws): the S3 code in `web/lib/s3.ts`. +- `block-kit` and `slack-api` (slackapi/slack-skills-plugin): Slack messages and Web API calls in `web/lib/slack/`. +- `vercel-functions`, `env-vars`, and `access-protected-vercel-deployment` (vercel/vercel-plugin): the cron job in `web/vercel.json`, environment variables, and testing protected preview deployments. diff --git a/web/CLAUDE.md b/web/CLAUDE.md deleted file mode 100644 index 43c994c2..00000000 --- a/web/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -@AGENTS.md diff --git a/web/components/AGENTS.md b/web/components/AGENTS.md new file mode 100644 index 00000000..70ed64bd --- /dev/null +++ b/web/components/AGENTS.md @@ -0,0 +1,20 @@ +# Components + +## Accessibility + +- Dialogs return focus to the control that opened them. +- Accessible names contain the visible label text. +- A control that is disabled for a reason stays focusable with `aria-disabled` and says why. +- Enter and exit animations use the `animate-in` and `animate-out` classes, so the reduced-motion rule in `web/app/globals.css` turns them off. + +## Component APIs + +- Add a named `variant` with `cva`, or a separate component, instead of a new boolean prop. +- Build from the shadcn primitives in `web/components/ui/` before you write new ones, and add a new primitive with `npx shadcn add ` in `web/`. + +## Skills + +- `vercel-composition-patterns` (vercel-labs/agent-skills): component APIs, compound components, and variants. +- `web-design-guidelines` (vercel-labs/agent-skills): reviewing UI for accessibility and interface quality. +- `shadcn` (shadcn/ui): adding and composing shadcn components. +- `tanstack-form`, pinned at https://github.com/tanstack-skills/tanstack-skills/tree/6666c1b363325fdd079dd30fa07f6ea7ae4608b5/plugins/tanstack-form/skills/tanstack-form: forms built with `@tanstack/react-form`. diff --git a/web/lib/api/AGENTS.md b/web/lib/api/AGENTS.md new file mode 100644 index 00000000..101fc45d --- /dev/null +++ b/web/lib/api/AGENTS.md @@ -0,0 +1,6 @@ +# REST API + +- Routes that watchers or Lambda call use `authorizeToken`, never `authorize`, because a browser session carries every scope. Add each one to `MACHINE_ROUTES` in `web/tests/integration/machine-routes-reject-session.test.ts`. +- For a public endpoint, register its schemas and path under `web/lib/api/openapi/` and import the path module in `web/lib/api/openapi/document.ts`. Without that import, the endpoint is missing from the spec and the docs. Routes that only the web UI uses stay out of OpenAPI. +- A server change that older watchers can't handle goes behind a feature in `WATCHER_FEATURES` in `web/lib/api/watcher-compat.ts`, and keeps the old behavior when the version header is missing. Lab PCs keep running old watcher versions for weeks. Remove the gate once `min_supported_version` reaches it. +- When you change the `lambda` preset in `web/lib/api/scope-catalog.ts`, update `_LAMBDA_PRESET_SCOPES` in `lambda/tests/integration/test_lambda_api.py` to match, and mint a new Lambda token before you deploy. diff --git a/web/lib/db/AGENTS.md b/web/lib/db/AGENTS.md new file mode 100644 index 00000000..9b216045 --- /dev/null +++ b/web/lib/db/AGENTS.md @@ -0,0 +1,9 @@ +# Database schema + +- After you edit `web/lib/db/schema.ts`, run `npm run db:generate` in `web/` and commit the SQL and `meta/` files it writes under `web/drizzle/`. Local development applies the schema with `db:push`. `staging` and `production` apply `web/drizzle/` through `.github/workflows/apply-migrations.yml` when the change merges. +- Never edit a migration after it merges, because environments that already applied it won't run it again. Write a new migration instead. +- A migration that creates an extension or collation needs the same statement in `web/scripts/reset-database.ts` and `web/tests/integration/global-setup.ts`, because `db:push` doesn't run migrations. +- Never hand-edit `web/lib/db/auth-schema.ts`. Run `npm run db:generate-auth-schema` in `web/` instead. The Better Auth skill's `npx auth generate` skips this repo's changes to that file. +- Don't drop a column in the same pull request that stops using it. Drop it in a later one, so rolling back the first still works. + +Drizzle docs: https://orm.drizzle.team/llms.txt diff --git a/web/lib/mcp/AGENTS.md b/web/lib/mcp/AGENTS.md new file mode 100644 index 00000000..a7399f9b --- /dev/null +++ b/web/lib/mcp/AGENTS.md @@ -0,0 +1,13 @@ +# MCP server + +Read `docs/developer/mcp-docs.md` before you change the docs or changelog tools. + +- Adding or removing a tool changes the tool count pinned in `web/tests/mcp/mcp-catalog.test.ts`, `web/tests/mcp/mcp-protocol.test.ts` (including `EXPECTED_TOOLS`), and `web/tests/integration/mcp.test.ts`. Update all three in the same commit. +- `*.output.ts` files use plain Zod. Don't import schemas that call `.openapi()`, such as `web/lib/api/openapi/schemas/runs.ts`, because those need registry side effects that unit tests skip. +- Pass a tool's `outputSchema` through whole, and don't wrap it in `z.object()`. Discriminated unions, such as the one `get_run_archive` returns, must stay intact. +- Keep `MCP_SERVER_INSTRUCTIONS` in `web/lib/mcp/instructions.ts` at or under 2 KB, with the most important points first. Claude Code keeps only the first 2 KB, and a unit test enforces the limit. +- MCP accepts OAuth access tokens only, not personal access tokens. Tools act as the signed-in person, so take user IDs from the auth context, never from tool arguments. + +MCP docs: https://modelcontextprotocol.io/llms.txt + +Skill: `mcp-builder` (anthropics/skills), for designing tools and writing their descriptions. diff --git a/web/mcp-apps/AGENTS.md b/web/mcp-apps/AGENTS.md new file mode 100644 index 00000000..4db293bc --- /dev/null +++ b/web/mcp-apps/AGENTS.md @@ -0,0 +1,9 @@ +# MCP Apps View + +Read `docs/developer/mcp-apps.md` before you change the View. + +`make dev` doesn't rebuild the View. After you change files here, run `npm run mcp-apps:build` in `web/`. Its output in `web/mcp-apps/dist/` is gitignored, and `npm run build` runs this step before `next build`. + +MCP Apps docs: https://modelcontextprotocol.io/extensions/apps/overview.md + +Skill: `add-app-to-server` (modelcontextprotocol/ext-apps), for adding UI to an MCP tool with `@modelcontextprotocol/ext-apps`. diff --git a/web/next.config.mjs b/web/next.config.mjs index 23485b5f..d1f24ce7 100644 --- a/web/next.config.mjs +++ b/web/next.config.mjs @@ -6,6 +6,10 @@ const appDir = path.dirname(fileURLToPath(import.meta.url)); /** @type {import('next').NextConfig} */ const nextConfig = { + // `web/AGENTS.md` carries its own Next.js guidance. Without this, every + // `next dev` writes its managed block back into that file. + agentRules: false, + // The MCP server moved from `/api/v1/mcp` to `/mcp/v1`. MCP client configs // (Claude Desktop, Cursor, …) live on end users' machines and can't be // migrated for them, so the old paths keep working. 308 preserves the POST diff --git a/web/scripts/docs-screenshots/environment.ts b/web/scripts/docs-screenshots/environment.ts index 4f7e7451..1d433fc4 100644 --- a/web/scripts/docs-screenshots/environment.ts +++ b/web/scripts/docs-screenshots/environment.ts @@ -43,7 +43,7 @@ const DEFAULT_PG_URL = "postgres://postgres:postgres@127.0.0.1:5432"; export const DATABASE_NAME = "data_hub_screenshots"; // Files in `web/` that `next dev` edits when it starts. -const REWRITTEN_BY_NEXT = ["next-env.d.ts", "tsconfig.json", "AGENTS.md"]; +const REWRITTEN_BY_NEXT = ["next-env.d.ts", "tsconfig.json"]; // Fixed so repeat runs show the same dates and times regardless of the // machine's own time zone. The browser context uses the same one. diff --git a/web/tests/AGENTS.md b/web/tests/AGENTS.md new file mode 100644 index 00000000..eeab2563 --- /dev/null +++ b/web/tests/AGENTS.md @@ -0,0 +1,6 @@ +# Web tests + +- Next.js returns a relative `Location` header for same-site redirects. Parse it with `new URL(location, getBaseUrl())`. +- Use your own test data, not the text of real changelog entries or docs pages, so content edits don't break tests. +- The MCP OAuth endpoints are rate-limited per IP, so call `getMcpAccessToken` once per user in `beforeAll`, not in each test. +- Reset the database with `resetDb()` from `web/tests/integration/helpers.ts`. It keeps the MCP signing keys and reseeds the single-row config tables.