diff --git a/.changeset/concurrent-boot-artifact-race.md b/.changeset/concurrent-boot-artifact-race.md new file mode 100644 index 000000000..8f51abf21 --- /dev/null +++ b/.changeset/concurrent-boot-artifact-race.md @@ -0,0 +1,5 @@ +--- +"everything-dev": patch +--- + +Concurrent `bos dev` sessions in one config dir no longer crash artifact generation: `.bos/bos.resolved-config.json` is now written atomically (tmp + rename) so a sibling session never reads a torn file, and the contract-bridge stale sweep removes gen files with `rmSync(…, { force: true })` so a file vanishing between the existence check and the removal no longer fails the boot with `ENOENT … lstat`. diff --git a/.changeset/pglite-only-local-db.md b/.changeset/pglite-only-local-db.md new file mode 100644 index 000000000..8aac6936e --- /dev/null +++ b/.changeset/pglite-only-local-db.md @@ -0,0 +1,5 @@ +--- +"everything-dev": minor +--- + +PGlite is the only local database — docker-compose is retired (ADR 0038). The zero-config dev default is now a persistent, file-backed PGlite data dir (`pglite:.bos/db`, per workspace process), so concurrent local projects can never collide on DB ports and data survives restarts without any `.env`. `bos dev` migrates leftovers once: compose-era `postgres://everythingdev:…@localhost:543x/…` URLs in `.env` are rewritten to the pglite defaults and a scaffold-shaped `docker-compose.yml` is relocated to `.bos/compose-retired/`. `db:*` commands work against pglite (drizzle-kit pglite dialect selected by URL scheme); the CLI/scaffold drop the compose template, `dev:postgres*` scripts, post-init docker prompt, and preflight auto-start; regression tests and the deploy smoke run on pglite (`TEST_DATABASE=postgres` + an external URL remains the parity escape hatch). Production `postgres://` URLs are unchanged. diff --git a/.env.example b/.env.example index 3f925a110..17767f1b2 100644 --- a/.env.example +++ b/.env.example @@ -10,26 +10,13 @@ CORS_ORIGIN=http://localhost:3000 # app.api -# Leave empty to run dev on ephemeral pglite; set a postgres:// URL to opt into compose Postgres. +# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access. API_DATABASE_URL= - -# apps/upload (standalone upload service — BOS_STORAGE_* are service-side env, -# not composed-plugin secrets; the deploy CLI uses BOS_STORAGE_ORIGIN + -# BOS_STORAGE_API_KEY from its own env/session, see .github/workflows/deploy.yml) -BOS_STORAGE_ENDPOINT= -BOS_STORAGE_BUCKET= -BOS_STORAGE_ACCESS_KEY_ID= -BOS_STORAGE_SECRET_ACCESS_KEY= -BOS_STORAGE_REGION= -BOS_STORAGE_AUTH_ORIGIN=http://localhost:3000 -BOS_STORAGE_BOOTSTRAP_API_KEY= -BOS_STORAGE_BOOTSTRAP_ACCOUNT= -BOS_STORAGE_BOOTSTRAP_GATEWAY= +BETTER_AUTH_SECRET= # app.auth -# Leave empty to run dev on ephemeral pglite; set a postgres:// URL to opt into compose Postgres. +# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access. AUTH_DATABASE_URL= -BETTER_AUTH_SECRET= GITHUB_CLIENT_SECRET= GOOGLE_CLIENT_SECRET= FASTNEAR_API_KEY= diff --git a/.env.test b/.env.test index 7802def49..28a9bcdb8 100644 --- a/.env.test +++ b/.env.test @@ -1,19 +1,19 @@ # Generated test environment — loaded by test suites instead of .env -# Test databases run on dedicated *-test docker-compose services +# Test databases run on in-memory pglite — no docker involved # app.api -API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db +API_DATABASE_URL=pglite::memory: BETTER_AUTH_SECRET=regression-test-secret-do-not-use-in-production # app.auth -AUTH_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5435/auth_test_db +AUTH_DATABASE_URL=pglite::memory: # plugins.template -TEMPLATE_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db +TEMPLATE_DATABASE_URL=pglite::memory: # plugins.proposals -PROPOSALS_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db +PROPOSALS_DATABASE_URL=pglite::memory: # plugins.votes -VOTES_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db +VOTES_DATABASE_URL=pglite::memory: diff --git a/.github/templates/api/drizzle.config.ts b/.github/templates/api/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/.github/templates/api/drizzle.config.ts +++ b/.github/templates/api/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/.github/templates/api/plugin.dev.ts b/.github/templates/api/plugin.dev.ts index a180097c4..0e6403510 100644 --- a/.github/templates/api/plugin.dev.ts +++ b/.github/templates/api/plugin.dev.ts @@ -26,7 +26,7 @@ export default { variables: {}, // child-owned-end secrets: { - API_DATABASE_URL: process.env.API_DATABASE_URL || "pglite:.bos/api/:memory:", + API_DATABASE_URL: process.env.API_DATABASE_URL || "pglite:.bos/db", BETTER_AUTH_SECRET: devAuthSecret, }, } satisfies PluginConfigInput, diff --git a/.github/templates/api/src/index.ts b/.github/templates/api/src/index.ts index e0dd5002e..d7089836b 100644 --- a/.github/templates/api/src/index.ts +++ b/.github/templates/api/src/index.ts @@ -12,8 +12,10 @@ export default createPlugin({ secrets: z.object({ API_DATABASE_URL: z .string() - .default("pglite:.bos/api/:memory:") - .describe("Database connection string. Use pglite: for local, postgres:// for production."), + .default("pglite:.bos/db") + .describe( + "Database connection string. pglite: is a persistent local data dir (.bos/db); postgres:// for production or shared access.", + ), }), context: ContextSchema, diff --git a/.github/templates/docker-compose.yml b/.github/templates/docker-compose.yml deleted file mode 100644 index 1544bd278..000000000 --- a/.github/templates/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Static local development databases — managed by hand, never generated. -# If a host port is already taken, change the left side of the ports mapping -# and update the matching *_DATABASE_URL in .env to match. -# -# Child runtimes develop the API (and optionally host) locally; auth is -# consumed from the extended platform, so no auth database is needed here. -services: - postgres-api: - image: public.ecr.aws/docker/library/postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_db - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d api_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_api_data:/var/lib/postgresql/data - - postgres-api-test: - image: public.ecr.aws/docker/library/postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - "5434:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d api_test_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_api_test_data:/var/lib/postgresql/data - -volumes: - postgres_api_data: - postgres_api_test_data: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de3f05354..0356509c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -515,37 +515,8 @@ jobs: timeout-minutes: 15 env: CI: true - API_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5434/api_test_db - AUTH_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5435/auth_test_db CORS_ORIGIN: http://localhost:4100 BETTER_AUTH_SECRET: regression-test-secret-do-not-use-in-production - services: - postgres-api-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - 5434:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d api_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 - postgres-auth-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_test_db - ports: - - 5435:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d auth_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 steps: - name: Checkout repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 (Node 24) @@ -608,38 +579,9 @@ jobs: timeout-minutes: 20 env: CI: true - API_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5434/api_test_db - AUTH_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5435/auth_test_db CORS_ORIGIN: http://localhost:4100 BETTER_AUTH_SECRET: regression-test-secret-do-not-use-in-production REGRESSION_PREBUILT_IMAGE: ${{ needs.regression-image.result == 'success' && needs.regression-image.outputs.image || '' }} - services: - postgres-api-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - 5434:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d api_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 - postgres-auth-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_test_db - ports: - - 5435:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d auth_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 steps: - name: Checkout repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 (Node 24) @@ -789,37 +731,8 @@ jobs: timeout-minutes: 20 env: CI: true - API_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5434/api_test_db - AUTH_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5435/auth_test_db CORS_ORIGIN: http://localhost:4100 BETTER_AUTH_SECRET: regression-test-secret-do-not-use-in-production - services: - postgres-api-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - 5434:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d api_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 - postgres-auth-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_test_db - ports: - - 5435:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d auth_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 steps: - name: Checkout repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 (Node 24) @@ -929,38 +842,9 @@ jobs: timeout-minutes: ${{ matrix.timeout }} env: CI: true - API_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5434/api_test_db - AUTH_DATABASE_URL: postgres://everythingdev:everythingdev@127.0.0.1:5435/auth_test_db CORS_ORIGIN: http://localhost:4100 BETTER_AUTH_SECRET: regression-test-secret-do-not-use-in-production REGRESSION_PREBUILT_IMAGE: ${{ needs.regression-image.result == 'success' && needs.regression-image.outputs.image || '' }} - services: - postgres-api-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - 5434:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d api_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 - postgres-auth-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 - env: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_test_db - ports: - - 5435:5432 - options: >- - --health-cmd "pg_isready -U everythingdev -d auth_test_db" - --health-interval 3s - --health-timeout 3s - --health-retries 10 steps: - name: Checkout repository uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 (Node 24) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 51a1f65db..4429aed6a 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -286,12 +286,6 @@ jobs: - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - # The post-publish smoke boots the runtime image identity-driven; its - # boot surface needs the compose test databases (the smoke-container - # idiom: host.docker.internal:5434/5435). - - name: Start test databases - run: docker compose up -d --wait postgres-api-test postgres-auth-test - - name: Deploy (preflight → build → upload → publish → smoke → Railway pin) timeout-minutes: 55 # One command runs the deploy train (ADR 0020/0021, image leg split diff --git a/.github/workflows/mirror-images.yml b/.github/workflows/mirror-images.yml index 18c867b3c..2b4352a79 100644 --- a/.github/workflows/mirror-images.yml +++ b/.github/workflows/mirror-images.yml @@ -3,10 +3,10 @@ name: Mirror base images # Docker Hub and ECR Public both rate-limit GitHub-hosted runners' shared # egress IPs (Docker Hub 504s motivated the ECR move; ECR then throttled with # `toomanyrequests: Data limit exceeded`, killing jobs before setup). The -# org's GHCR mirror is the pull source for the three library base images; +# org's GHCR mirror is the pull source for the library base images; # this workflow refreshes it from upstream. Renovate tracks the mirrors and # PRs the digest bumps. New GHCR packages land private — flip them public in -# package settings once (local dev docker compose pulls them unauthenticated). +# package settings once. on: workflow_dispatch: @@ -27,9 +27,6 @@ jobs: - source: public.ecr.aws/docker/library/node:24-alpine fallback: docker.io/library/node:24-alpine target: ghcr.io/nearbuilders/mirror-node:24-alpine - - source: public.ecr.aws/docker/library/postgres:17-alpine - fallback: docker.io/library/postgres:17-alpine - target: ghcr.io/nearbuilders/mirror-postgres:17-alpine - source: public.ecr.aws/docker/library/redis:7-alpine fallback: docker.io/library/redis:7-alpine target: ghcr.io/nearbuilders/mirror-redis:7-alpine diff --git a/AGENTS.md b/AGENTS.md index 28c169418..0ffd47a7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -239,25 +239,37 @@ This document provides operational guidance for AI agents working in the parent pnpm install pnpm run dev -# Zero-config: no .env needed — plugins boot on ephemeral pglite, compose stays down, -# and BETTER_AUTH_SECRET is generated per-run (dev-only, not persisted). -# Opt into persistent compose Postgres by setting the *_DATABASE_URL URLs in .env -# (copy .env.example); the DB preflight then probes them and auto-starts compose. -# Or combined: pnpm run dev:postgres == docker compose up -d --wait && pnpm run dev +# Zero-config: no .env needed — the local database is a persistent, file-backed +# PGlite data dir (.bos/db per workspace, ADR 0038); BETTER_AUTH_SECRET is +# generated per-run (dev-only, not persisted). Opt out of the default by writing +# a postgres:// URL into *_DATABASE_URL in .env (production/shared server). # Pin individual service ports (explicitly-passed flags are pinned; unset services derive from the base; only explicit choices persist in .bos/infra-state.json — see ADR 0012) bos dev --port 3100 --api-port 3101 --ui-port 3103 --auth-port 3102 --plugin-port-start 3110 ``` -`docker-compose.yml` is committed and provisions four Postgres 17 services — two for dev, two for tests: -- `postgres-api` (port 5432, db `api_db`) — shared by the API and all local plugins; the API keeps its tables in `public`, while each plugin isolates its tables in a `plugin_` schema (set via `search_path` on every connection, see `api/src/db/layer.ts`). -- `postgres-auth` (port 5433, db `auth_db`) — auth database. -- `postgres-api-test` (port 5434, db `api_test_db`) — test-only API/plugin database. -- `postgres-auth-test` (port 5435, db `auth_test_db`) — test-only auth database. - -**Dev/test isolation:** a committed `.env.test` (generated alongside `.env.example` and `docker-compose.yml` by `bos dev`) maps every `*_DATABASE_URL` and `BETTER_AUTH_SECRET` to the test databases. Test suites load `.env.test` instead of `.env`: the regression stack (`tests/regression/lib/start-stack.mjs`) injects it into spawned stacks, `tests/regression/lib/regression-env.mjs` resolves it with fail-fast guards that refuse to run against dev URLs or the dev auth secret (override deliberately with `REGRESSION_ALLOW_DEV_DB=1`), and api unit/integration tests pin to in-memory pglite unless `TEST_DATABASE=postgres` opts into `.env.test`. Start the test databases with `pnpm run test:db:up` (or `pnpm run test:db:reset` for a clean slate). - -The API and plugins auto-apply migrations on boot, so `pnpm run db:migrate` is optional (use it to migrate without starting the dev server). `pnpm run dev` runs `bos dev`'s preflight, which probes the localhost DB ports and — when every failure is a down local service, `docker-compose.yml` exists, docker is reachable, and it is not a test-mode stack — starts the compose services itself (`docker compose up -d --wait`) and re-probes once before failing. Test-mode stacks (`NODE_ENV=test` / `BOS_TEST=1` / `BOS_NO_PERSIST_PORTS=1`) never auto-start compose. Bootstrap-phase INFO logs (e.g. `[env] ... updated` port-drift lines) are suppressed on the console by default — pass `--log-level info` (or set `BOS_LOG_LEVEL` / `DEBUG=1`) to see them; warnings and errors always print. +**Databases — PGlite only (ADR 0038):** no docker-compose, no local Postgres. Each +workspace process persists to its own `pglite:.bos/db` data dir (`api/.bos/db`, +`plugins//.bos/db`, …); concurrent projects (even 40 at once) can never fight over DB +ports. The api and its in-process plugins share one data dir with `plugin_` schema +isolation; auth keeps its own dir. pglite ≥ 0.4.6 supports multiple instances per data dir +across processes, so `db:studio`/`db:migrate`/`db:push` work while dev runs (they may queue +behind dev's writes; two dev sessions in one repo share data). + +**Degraded/parity mode:** production or shared-server access points `*_DATABASE_URL` in +`.env` at a real `postgres://` URL (Neon/Railway per ADR 0035). Parity testing pglite/pg +divergence sets `TEST_DATABASE=postgres` + the external URL — no committed compose exists. + +**Dev/test isolation:** a committed `.env.test` (generated by `bos dev`) maps every +`*_DATABASE_URL` to in-memory pglite and `BETTER_AUTH_SECRET` to the test secret. Test +suites load `.env.test` instead of `.env`: the regression stack +(`tests/regression/lib/start-stack.mjs`) injects it into spawned stacks, +`tests/regression/lib/regression-env.mjs` resolves it with fail-fast guards that refuse +to run against dev data (the per-workspace `.bos/db` default or legacy dev postgres URLs; +override deliberately with `REGRESSION_ALLOW_DEV_DB=1`), and api unit/integration tests +pin to in-memory pglite unless `TEST_DATABASE=postgres` opts into external URLs. + +The API and plugins auto-apply migrations on boot, so `pnpm run db:migrate` is optional (use it to migrate without starting the dev server). `pnpm run dev` runs `bos dev`'s preflight, which probes any `postgres://`/`redis://` URLs the operator configured (pglite URLs need no probe) before spawning. The first run also migrates state left over from the retired docker-compose era: conventional `postgres://everythingdev:…@localhost:543x/…` URLs in `.env` are rewritten to the pglite defaults and a scaffold-shaped `docker-compose.yml` is relocated to `.bos/compose-retired/`. Bootstrap-phase INFO logs (e.g. `[env] ... updated` port-drift lines) are suppressed on the console by default — pass `--log-level info` (or set `BOS_LOG_LEVEL` / `DEBUG=1`) to see them; warnings and errors always print. Port allocation is atomic block allocation (ADR 0012): the layout derives deterministically from one base port (`--port N` → api N+1, auth N+2, ui N+3, plugins N+10+), the whole block is validated before anything spawns, and only explicitly-passed port flags persist to `.bos/infra-state.json` under `devPorts` (drift is never persisted — a busy block moves +100 with a prominent notice naming the holders, and restarts re-try the preferred base). Explicitly-passed flags are pinned: if that exact port is occupied, allocation fails loudly instead of silently moving. `bos kill` escalates SIGTERM → 5s → SIGKILL, reaps orphaned children of dead sessions, and verifies ports actually freed; new sessions adopt-and-reap orphaned children from dead same-project sessions at boot. Test-spawned stacks never persist ports (`BOS_NO_PERSIST_PORTS=1`, plus `NODE_ENV=test` / `BOS_TEST=1` are honored), so test runs can never repin your dev ports. `CORS_ORIGIN` in `.env.example` is derived from the actual resolved host port in development. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 61d0d27c3..df67152ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,13 +14,13 @@ New here? Get onboarded and plugged in: ## Quick Setup ```bash -bun install # Install dependencies -bun run dev # Start development (host mode auto-detected) +pnpm install # Install dependencies +pnpm run dev # Start development (host mode auto-detected) ``` -That's it — on first run `bos dev` creates `.env` from `.env.example` (with a generated `BETTER_AUTH_SECRET`), and if local Postgres is down it starts it for you via `docker compose up -d --wait`. Docker must be installed and running. +That's it — no Postgres, no Docker. On first run `bos dev` creates `.env` from `.env.example` (with a generated `BETTER_AUTH_SECRET`); every local database is a persistent PGlite data dir under `.bos/db` (ADR 0038). Set a `postgres://` URL in `.env` only when you need production-style or shared server access. -`bun run dev:postgres` starts the databases explicitly; `dev:postgres:down` / `dev:postgres:reset` manage them. `bun db:migrate` is optional — the API and plugins auto-apply migrations on boot. +`pnpm run db:migrate` is optional — the API and plugins auto-apply migrations on boot. Visit http://localhost:3003 (UI), http://localhost:3001 (API), and http://localhost:3002 (Auth). @@ -128,7 +128,7 @@ We use [Changesets](https://github.com/changesets/changesets) for versioning. **Create a changeset:** ```bash -bun run changeset +pnpm run changeset # Follow prompts to select packages and write description ``` @@ -151,9 +151,9 @@ Added new endpoint for user profiles 1. **Before creating PR:** ```bash - bun run test # Run all tests - bun typecheck # Type check all packages - bun lint # Run linting + pnpm run test # Run all tests + pnpm run typecheck # Type check all packages + pnpm run lint # Run linting ``` 2. **Create PR from your fork:** @@ -178,8 +178,8 @@ Added new endpoint for user profiles 2. **Clone** your fork locally 3. **Create** a feature branch: `git checkout -b feature/amazing-feature` 4. **Make** your changes -5. **Test** thoroughly: `bun run test` and `bun typecheck` -6. **Add changeset** if needed: `bun run changeset` +5. **Test** thoroughly: `pnpm run test` and `pnpm run typecheck` +6. **Add changeset** if needed: `pnpm run changeset` 7. **Commit** using [Semantic Commits](https://gist.github.com/joshbuchea/6f47e86d2510bce28f8e7f42ae84c716) 8. **Push** to your fork: `git push origin feature/amazing-feature` 9. **Open** a Pull Request to the main repository @@ -198,9 +198,9 @@ Added new endpoint for user profiles We use [Biome](https://biomejs.dev/) for linting and formatting: ```bash -bun lint # Check linting -bun lint:fix # Fix auto-fixable issues -bun format # Format code +pnpm run lint # Check linting +pnpm run lint --fix # Fix auto-fixable issues +pnpm run format # Format code ``` ## Reporting Issues diff --git a/api/drizzle.config.ts b/api/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/api/drizzle.config.ts +++ b/api/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/api/plugin.dev.ts b/api/plugin.dev.ts index a180097c4..0e6403510 100644 --- a/api/plugin.dev.ts +++ b/api/plugin.dev.ts @@ -26,7 +26,7 @@ export default { variables: {}, // child-owned-end secrets: { - API_DATABASE_URL: process.env.API_DATABASE_URL || "pglite:.bos/api/:memory:", + API_DATABASE_URL: process.env.API_DATABASE_URL || "pglite:.bos/db", BETTER_AUTH_SECRET: devAuthSecret, }, } satisfies PluginConfigInput, diff --git a/api/src/index.ts b/api/src/index.ts index d48e0aa66..4930e1bcc 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -66,8 +66,10 @@ export default createPlugin.withPlugins()({ secrets: z.object({ API_DATABASE_URL: z .string() - .default("pglite:.bos/api/:memory:") - .describe("Database connection string. Use pglite: for local, postgres:// for production."), + .default("pglite:.bos/db") + .describe( + "Database connection string. pglite: is a persistent local data dir (.bos/db); postgres:// for production or shared access.", + ), BETTER_AUTH_SECRET: z .string() .describe( diff --git a/bos.app.ts b/bos.app.ts index cde7929a9..9699faba0 100644 --- a/bos.app.ts +++ b/bos.app.ts @@ -38,6 +38,9 @@ export default App({ "TWILIO_AUTH_TOKEN", "TWILIO_PHONE_NUMBER", "RESEND_API_KEY", + // Dev/test-only seeding seam (ADR 0038 addendum) — declared so the + // regression stacks' env reaches the plugin; never set in production. + "AUTH_BOOTSTRAP_ADMIN_API_KEY", ], variables: { organizationMembershipLimit: 1000, diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ab4525553..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Static local development databases — managed by hand, never generated. -# If a host port is already taken, change the left side of the ports mapping -# and update the matching *_DATABASE_URL in .env to match. -# -# Volume migration (one-time, from older account-prefixed compose files): -# the volumes below are scoped to this directory's project name. To keep -# data from a previous generated file, copy it first, e.g.: -# docker run --rm -v v1_citynode_near_postgres_api_data:/from \ -# -v _postgres_api_data:/to alpine sh -c "cp -a /from/. /to/" -services: - postgres-api: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_db - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d api_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_api_data:/var/lib/postgresql/data - - postgres-auth: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_db - ports: - - "5433:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d auth_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_auth_data:/var/lib/postgresql/data - - postgres-api-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: api_test_db - ports: - - "5434:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d api_test_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_api_test_data:/var/lib/postgresql/data - - postgres-auth-test: - image: ghcr.io/nearbuilders/mirror-postgres:17-alpine - environment: - POSTGRES_USER: everythingdev - POSTGRES_PASSWORD: everythingdev - POSTGRES_DB: auth_test_db - ports: - - "5435:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready -U everythingdev -d auth_test_db"] - interval: 3s - timeout: 3s - retries: 5 - volumes: - - postgres_auth_test_data:/var/lib/postgresql/data - -volumes: - postgres_api_data: - postgres_auth_data: - postgres_api_test_data: - postgres_auth_test_data: diff --git a/docs/adr/0038-pglite-only-local-db.md b/docs/adr/0038-pglite-only-local-db.md new file mode 100644 index 000000000..c47d29ab6 --- /dev/null +++ b/docs/adr/0038-pglite-only-local-db.md @@ -0,0 +1,87 @@ +# 0038 — PGlite is the only local database + +Date: 2026-10-30 · Status: accepted + +## Context + +Every scaffolded app shipped a `docker-compose.yml` with hardcoded host ports +5432–5435. Concurrent projects each tried to bind the same block, so starting a +second `bos dev` failed with `Bind for 0.0.0.0:5433 failed: port is already +allocated` (observed between `nearbuilders.org` and `work.efiz.near`). The +zero-config default was *ephemeral* pglite (`pglite:.bos//:memory:`, +plan 055 ticket 11) — data lost on every restart — which is exactly why +operators opted into compose Postgres just to persist data. The conflict class +was structural: any operator wanting persistence had to race every other +operator for one port block. + +Two real options existed: per-project DB port allocation (extend ADR 0012), or +a single shared postgres with per-project databases. Both keep a server, ports, +docker lifecycle, and credentials in the dev-loop critical path. + +## Decision + +**Local persistence is a file-backed PGlite data dir; docker-compose is +retired entirely (dev and test).** + +- The zero-config dev default is `pglite:.bos/db` — persistent, per + workspace process (processes spawn with cwd = the workspace, so `api/`, + `plugins//`, each keep their own data dir; no ports, no docker, no + cross-project collision is possible by construction). +- The two-database contract is unchanged: auth keeps its own data dir; api and + its in-process plugins share one dir, with plugins schema-isolated under + `plugin_` (the fallback `*_DATABASE_URL` → `API_DATABASE_URL` now + yields genuine shared data instead of disjoint in-memory islands). +- Tests run on in-memory pglite (`pglite::memory:` / temp dirs). `.env.test` + carries only the test auth secret + in-memory URLs. +- `TEST_DATABASE=postgres` + an explicit external `postgres://` URL remains + the parity escape hatch (pglite/postgres savepoint semantics diverge — + plan 008 findings). No committed compose anywhere; CI keeps no postgres. +- `bos dev` repairs legacy state once: conventional compose-era URLs + (`postgres://everythingdev:…@localhost:{5432–5435}/{api_db,auth_db,…}`, + including drift ports) in `.env` are rewritten to the new defaults, and a + scaffold-shaped `docker-compose.yml` is relocated to `.bos/compose-retired/`. + Operator-authored postgres URLs and compose files are untouched. +- Production is unchanged: `postgres://` URLs, Neon/Railway, ADR 0035. + +## Consequences + +- **Port conflicts are structurally impossible** for the local default — the + collision class this ADR exists for cannot recur. +- **Concurrent access works out of the box.** pglite ≥ 0.4.6 supports multiple + instances per data dir across processes (multi-worker file locking), + verified empirically in this repo: a second process reads what a first + wrote, and `db:studio`/`db:migrate` run while `bos dev` holds the dir. Two + dev sessions in one repo share data rather than failing — heavy concurrent + writers queue on the file lock, which is acceptable at dev scale. +- **drizzle-kit needs the `pglite` dialect** for `pglite:` URLs: configs now + select dialect/credentials via `drizzleCredentialsFor(url)` (`dialect: + "pglite"` + `driver: "pglite"`, data-dir path), plain postgres URLs keep + `dialect: "postgresql"`. +- **The driver creates the data dir recursively.** PGlite mkdirs only the leaf + component; a missing `.bos/` prefix previously failed ENOENT at boot. +- **Seek-time / heavy-query performance** is WASM-bound — acceptable for dev + workloads (the plan 055 zero-config decision already accepted this class of + tradeoff); production parity stays on real postgres. +- **The compose-era skill surface dies**: no `dev:postgres*`, `test:db:up`, + `test:db:reset` scripts, no compose template in `bos init`, no preflight + auto-start. ADR 0012's port-block model now governs only the booted + services (host/api/auth/ui/plugins), which was always its actual concern. +- **Regression fixtures seed through the API, not the DB.** With in-memory + pglite the stack's database lives inside its own process — no out-of-process + client can reach it. The direct-DB test helpers (`auth-test-instance`, + `migrate-test-db`, the Go approve step) are replaced by HTTP seeding + (`tests/regression/lib/http-seed.mjs`): anonymous sign-in mints sessions, + Better Auth routes create orgs, and admin actions run through the existing + `reviewOrganization` / `adminSetRole` routes using a boot-time bootstrap + admin API key (`AUTH_BOOTSTRAP_ADMIN_API_KEY`, dev/test-only — the auth + plugin ensures a platform admin + API key from that secret; never set in + production). Cookie artifacts (`cookies.json`, `admin-cookies.json`, + `logout-cookies.json`) keep the same locations, captured as Playwright + cookie snaps from the sign-in responses. + +## References + +- Plan 055 ticket 11 (zero-config dev), the seed of this decision. +- Issue #651 (contradictory v1 infra state) — this retires that bug class. +- ADR 0012 (ports as scoped resources), ADR 0033 (db seam), ADR 0035 (durable + Postgres). diff --git a/docs/agents/testing.md b/docs/agents/testing.md index 9f20cfd07..9b76e7e72 100644 --- a/docs/agents/testing.md +++ b/docs/agents/testing.md @@ -74,13 +74,14 @@ boundary above; until it lands, this document is the contract. the stack is fine; five separate one-test spec files are five stack boots. Batch journeys by theme. - **The cache is real**: `global-setup` seeds users/orgs once and writes - cookies to `.bos/regression/*.json`. Specs `useCookieHeader` — reuse the - cached session; mint your own only for a state the fixtures don't cover, and - route it through the shared auth instance, never a bespoke one - (`createAuthTestInstance` against the frozen upstream schema snapshot is the - supported path). + cookies to `.bos/regression/*.json`. Specs `injectCookies` — reuse the + cached session; mint a new state only when the fixtures don't cover it, and + route the seeding through the shared HTTP helpers + (`tests/regression/lib/http-seed.mjs` and the bootstrap-admin API key is + the supported path — the stack database is in-process pglite, so direct DB + writes are impossible). - **Serial by design**: `workers: 1`, `fullyParallel: false`, `retries: 1` — - specs share ports, databases, and the compose. Don't write a spec that + specs share ports and the seeded cookie cache. Don't write a spec that assumes a clean global state another spec might have breathed on; seed fixtures for per-spec state. - **The stall watchdog** fails after ~120 s of zero test progress: a spec that diff --git a/host/.env.example b/host/.env.example index f5d82332e..b0ee48d43 100644 --- a/host/.env.example +++ b/host/.env.example @@ -3,4 +3,5 @@ CORS_ORIGIN=http://localhost:3000 CSP_STRICT= # app.api -API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5432/api_db \ No newline at end of file +# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access. +API_DATABASE_URL= \ No newline at end of file diff --git a/host/README.md b/host/README.md index bda7e48a8..bb03a64a6 100644 --- a/host/README.md +++ b/host/README.md @@ -86,7 +86,7 @@ For the temporary publish registry, use `bos publish` or `bos publish --deploy`. | `API_SOURCE` | `local` or `remote` | Based on NODE_ENV | | `API_PROXY` | Proxy API requests to another host URL | - | | `NETWORK_ID` | Tenant account suffix resolution: `mainnet` or `testnet` | `mainnet` | -| `AUTH_DATABASE_URL` | PostgreSQL URL for auth | `postgres://everythingdev:everythingdev@localhost:5433/auth_db` | +| `AUTH_DATABASE_URL` | PostgreSQL URL for auth; defaults to the local persistent `pglite:.bos/db` PGlite (ADR 0038) | `pglite:.bos/db` | | `BETTER_AUTH_SECRET` | Secret used for session encryption and key derivation | - | | `CORS_ORIGIN` | Comma-separated allowed origins | Host + UI URLs | diff --git a/package.json b/package.json index a10ddda04..ac0acc696 100644 --- a/package.json +++ b/package.json @@ -58,11 +58,6 @@ "regression:start:ssr": "node tests/regression/lib/start-container.mjs ssr", "regression:start:csr": "node tests/regression/lib/start-container.mjs csr", "regression:backcompat": "node tests/regression/lib/start-stack.mjs backcompat", - "test:db:up": "docker compose up -d --wait postgres-api-test postgres-auth-test", - "test:db:reset": "docker compose rm -fsv postgres-api-test postgres-auth-test && docker compose up -d --wait postgres-api-test postgres-auth-test", - "dev:postgres": "docker compose up -d --wait && pnpm run dev", - "dev:postgres:down": "docker compose down", - "dev:postgres:reset": "docker compose down -v && docker compose up -d --wait", "test:regression:browser": "pnpm run test:regression:browser:dev:ssr && pnpm run test:regression:browser:dev:csr && pnpm run test:regression:browser:start:ssr && pnpm run test:regression:browser:start:csr && pnpm run test:regression:browser:backcompat", "test:regression:http": "pnpm run test:regression:http:dev && pnpm run test:regression:http:start && pnpm run test:regression:http:backcompat", "test:regression:dev:ssr": "pnpm run test:regression:http:dev && pnpm run test:regression:browser:dev:ssr", diff --git a/packages/everything-dev/skills/dev-workflow/SKILL.md b/packages/everything-dev/skills/dev-workflow/SKILL.md index f40527157..35a050d80 100644 --- a/packages/everything-dev/skills/dev-workflow/SKILL.md +++ b/packages/everything-dev/skills/dev-workflow/SKILL.md @@ -213,7 +213,7 @@ Process tracking uses `~/.cache/everything-dev/pids.json` (global, atomic writes | word | do | |---|---| -| `Infra preflight failed: …` | the DB preflight probes localhost ports and auto-runs `docker compose up -d --wait` when every failure is a down local service — if it still fails, start Postgres (`pnpm run dev:postgres`) and read the failure messages | +| `Infra preflight failed: …` | the preflight probes only operator-configured `postgres://`/`redis://` URLs (pglite defaults need no probe) — point the failing `*_DATABASE_URL` at a reachable URL, or clear it to fall back to the persistent `.bos/db` PGlite | | `PortAllocationError` on an explicit port | the pinned port is occupied and allocation fails loudly — free it or pick another; it never silently moves an explicit choice | | `+100` port-shift notice at startup | expected drift: the preferred block was busy, holders are named in the notice, nothing is persisted — restarts re-try the preferred base | | service stuck in `starting` | probe its readiness path directly (e.g. `curl http://localhost:3003/remoteEntry.js`) and read `.bos/logs/.log` | diff --git a/packages/everything-dev/skills/plugin-development/SKILL.md b/packages/everything-dev/skills/plugin-development/SKILL.md index def49a39f..3b4ba28f8 100644 --- a/packages/everything-dev/skills/plugin-development/SKILL.md +++ b/packages/everything-dev/skills/plugin-development/SKILL.md @@ -343,8 +343,8 @@ One ~15-line file, scaffolded once and owned by the plugin (ADR 0033): the plugi ```ts // everything-dev/db's databaseLayer supports: -// "pglite:.bos/your-plugin/:memory:" — in-memory PGlite (dev) -// "pglite:.bos/your-plugin/data" — persistent PGlite (dev) +// "pglite:.bos/db" — persistent PGlite data dir (local default, ADR 0038) +// "pglite::memory:" — in-memory PGlite (tests, throwaway) // "" — real PostgreSQL (prod) // namespace: true (default) scopes into plugin_; false keeps the // public schema (api shell) or the dedicated database (auth) @@ -367,10 +367,10 @@ Migrations run inside `DatabaseLive`'s scoped layer. The critical rule is to **b Each plugin manages its own database or schema. Connection strings are passed via secrets (`YOURPLUGIN_DATABASE_URL`). The convention: ``` -YOURPLUGIN_DATABASE_URL=pglite:.bos/your-plugin/:memory: +YOURPLUGIN_DATABASE_URL=pglite:.bos/db ``` -In production, set this to a real PostgreSQL connection string in the host's environment. +In production, set this to a real PostgreSQL connection string in the host's environment. (Local plugins without an explicit URL fall back to `API_DATABASE_URL` — the two-database contract — so the default above matters mainly for standalone plugin dev.) ## Calling Other Plugins diff --git a/packages/everything-dev/src/cli/commands/dev.ts b/packages/everything-dev/src/cli/commands/dev.ts index a0338224e..af37b8ea9 100644 --- a/packages/everything-dev/src/cli/commands/dev.ts +++ b/packages/everything-dev/src/cli/commands/dev.ts @@ -49,7 +49,6 @@ export const runDev = async (ctx: CommandContext): Promise => { build: "Building...", "resolve config": "Resolving config...", ports: "Finding available ports...", - "docker compose up": "Starting local Postgres...", "generate artifacts": "Generating code artifacts...", }; diff --git a/packages/everything-dev/src/cli/commands/init.ts b/packages/everything-dev/src/cli/commands/init.ts index 2324ccacf..32798d22a 100644 --- a/packages/everything-dev/src/cli/commands/init.ts +++ b/packages/everything-dev/src/cli/commands/init.ts @@ -1,7 +1,6 @@ import * as p from "@clack/prompts"; import type { InitOptions, InitResult, OverrideSection } from "../../contract"; import { fetchInitParent } from "../../handlers/init"; -import { runDockerComposeUp } from "../../infra/docker"; import { type ProgressEvent, pluginEvents } from "../../services/dev/progress"; import { colors } from "../../utils/theme"; import { exitCodeFor } from "../exit-codes"; @@ -92,7 +91,7 @@ export const runInit = async (ctx: CommandContext): Promise => { "personalize config": "Personalizing config...", "write snapshot": "Writing snapshot...", "resolve config": "Resolving config...", - "generate env/docker": "Generating environment config...", + "generate env": "Generating environment config...", "create env file": "Creating .env file...", "install dependencies": "Installing dependencies...", "generate types": "Generating types...", @@ -144,24 +143,5 @@ export const runInit = async (ctx: CommandContext): Promise => { } console.log(); - if (initInput.noInteractive !== true && !initInput.noInstall && result.targetDir) { - const shouldStartDocker = await p.confirm({ - message: "Run docker compose up -d --wait?", - initialValue: true, - }); - - if (shouldStartDocker === true) { - const dockerSpinner = p.spinner(); - dockerSpinner.start("Starting Docker services"); - const compose = await runDockerComposeUp(result.targetDir); - if (compose.ok) { - dockerSpinner.stop("Docker services ready"); - } else { - dockerSpinner.stop("Docker services not started"); - p.log.warn(`docker compose up -d --wait failed${compose.tail ? `:\n${compose.tail}` : ""}`); - } - } - } - return; }; diff --git a/packages/everything-dev/src/db/binding.ts b/packages/everything-dev/src/db/binding.ts index 4514a6964..74acbb9b4 100644 --- a/packages/everything-dev/src/db/binding.ts +++ b/packages/everything-dev/src/db/binding.ts @@ -3,30 +3,6 @@ import { dirname, join } from "node:path"; import { parse as parseDotenv } from "dotenv"; import type { WorkspaceIdentity } from "./identity"; -/** - * drizzle-kit subcommands that open a database connection. For everything - * else (generate, check) the connection string is parsed but never used, so a - * placeholder is safe. - */ -const DB_REQUIRING_COMMANDS: ReadonlySet = new Set([ - "push", - "pull", - "migrate", - "studio", - "up", -]); - -/** - * Whether the current process is a drizzle-kit invocation whose command needs - * a live database. Mirrors drizzle-kit's CLI layout: `drizzle-kit …`. - */ -export function requiresDatabaseConnection(argv: readonly string[] = process.argv): boolean { - const isDrizzleKit = typeof argv[1] === "string" && argv[1].includes("drizzle-kit"); - if (!isDrizzleKit) return false; - const command = argv[2]; - return typeof command === "string" && DB_REQUIRING_COMMANDS.has(command); -} - /** * Materialize the environment a spawned tool needs to reach the binding's * database: the canonical secret name mapped to the resolved URL. The process @@ -37,6 +13,31 @@ export function bindingEnv(binding: { secretName: string; url: string }): Record return { [binding.secretName]: binding.url }; } +/** + * The zero-config local database: a persistent, file-backed PGlite data dir + * resolved per workspace process (`.bos/db` relative to the workspace cwd). + * No ports, no docker — concurrent projects never collide. + */ +export const PGLITE_DEFAULT_URL = "pglite:.bos/db"; + +/** + * drizzle-kit dialect options for a resolved database URL: `pglite:` URLs use + * the `pglite` dialect with a `driver`-tagged data-dir path; anything else is + * a regular postgres DSN. + */ +export function drizzleCredentialsFor(url: string): { + dialect: "postgresql" | "pglite"; + dbCredentials: { url: string; driver?: "pglite" }; +} { + if (url.startsWith("pglite:")) { + return { + dialect: "pglite", + dbCredentials: { driver: "pglite", url: url.replace("pglite:", "") }, + }; + } + return { dialect: "postgresql", dbCredentials: { url } }; +} + function findEnvFile(startDir: string | undefined): string | undefined { if (!startDir) return undefined; let current = startDir; @@ -58,22 +59,20 @@ function findEnvFile(startDir: string | undefined): string | undefined { * 2. the canonical secret in the nearest `.env` walked up from the * workspace directory (existing environment wins — values are not * overridden, matching dotenv semantics) - * 3. loud failure when the drizzle-kit command needs a database - * 4. an in-memory pglite placeholder for commands that never connect - * (generate, check) and for non-drizzle-kit consumers + * 3. the persistent local default (`PGLITE_DEFAULT_URL`) — valid for + * connecting commands too, so `pnpm run db:migrate`/`db:studio` work + * zero-config against the workspace's `.bos/db` data dir * - * This replaces the previous silent `pglite:` fallback that masked missing - * configuration as cryptic driver errors downstream. + * drizzle-kit commands the default works for are unaffected; non-drizzle-kit + * consumers get the same persistent default. */ export function resolveDatabaseUrl( identity: WorkspaceIdentity, options: { env?: Record; - argv?: readonly string[]; } = {}, ): string { const env = options.env ?? process.env; - const argv = options.argv ?? process.argv; const direct = env[identity.secretName]; if (direct) return direct; @@ -85,17 +84,9 @@ export function resolveDatabaseUrl( const fromFile = parsed[identity.secretName]; if (fromFile) return fromFile; } catch { - // unreadable .env — fall through to loud failure / placeholder + // unreadable .env — fall through to the default } } - if (requiresDatabaseConnection(argv)) { - throw new Error( - `Missing ${identity.secretName} for workspace "${identity.slug}" — ` + - `required for "drizzle-kit ${argv[2]}". ` + - `Add ${identity.secretName} to your .env (see .env.example) or run via \`bos db\`.`, - ); - } - - return `pglite:.bos/${identity.slug}/:memory:`; + return PGLITE_DEFAULT_URL; } diff --git a/packages/everything-dev/src/db/bindings.ts b/packages/everything-dev/src/db/bindings.ts index bb7b1af29..f809ae2f0 100644 --- a/packages/everything-dev/src/db/bindings.ts +++ b/packages/everything-dev/src/db/bindings.ts @@ -1,6 +1,7 @@ import { resolve } from "node:path"; import { Context, Data, Effect, Layer } from "effect"; import type { RuntimeConfig } from "../types"; +import { PGLITE_DEFAULT_URL } from "./binding"; import { getDatabaseUrlSecretName, getMigrationStorage, pluginMigrationSlug } from "./core"; import { type WorkspaceIdentity, workspaceIdentityFromWorkspaceDir } from "./identity"; @@ -91,15 +92,28 @@ function resolvePluginBinding( const url = env[identity.secretName]; if (!url) { const declaredDatabaseSecrets = (secrets ?? []).filter((s) => s.endsWith("_DATABASE_URL")); - const mismatch = - declaredDatabaseSecrets.length > 0 && !declaredDatabaseSecrets.includes(identity.secretName) - ? ` Config declares ${declaredDatabaseSecrets.join(", ")} — the workspace package name and the declared secret disagree.` - : ""; - throw new Error( - `.env missing ${identity.secretName} for plugin "${key}".` + - mismatch + - ` Add it to your .env file (see .env.example).`, - ); + if ( + declaredDatabaseSecrets.length > 0 && + !declaredDatabaseSecrets.includes(identity.secretName) + ) { + throw new Error( + `.env missing ${identity.secretName} for plugin "${key}".` + + ` Config declares ${declaredDatabaseSecrets.join(", ")} — the workspace package name and the declared secret disagree.` + + ` Add it to your .env file (see .env.example).`, + ); + } + if (!identity.workspaceDir) { + throw new Error( + `.env missing ${identity.secretName} for remote plugin "${key}" — a remote binding needs an explicit URL; add it to your .env file.`, + ); + } + return { + key, + source: source ?? "remote", + section, + identity, + url: PGLITE_DEFAULT_URL, + }; } return { diff --git a/packages/everything-dev/src/db/driver.ts b/packages/everything-dev/src/db/driver.ts index 9ac1cdf78..c0325f0d6 100644 --- a/packages/everything-dev/src/db/driver.ts +++ b/packages/everything-dev/src/db/driver.ts @@ -1,5 +1,4 @@ import { mkdirSync } from "node:fs"; -import { dirname } from "node:path"; import type { PgDatabase, PgQueryResultHKT } from "drizzle-orm/pg-core"; import { pluginMigrationSlug } from "./core"; @@ -211,7 +210,8 @@ export async function createDatabaseDriver {} + +export interface LegacyMigrationReport { + databaseUrlDrift: Array<{ key: string; from: string; to: string }>; + composeRelocated: boolean; +} + +/** + * The compose-era local database convention (`bos dev` provisioned Postgres + * via the committed docker-compose.yml): every `*_DATABASE_URL` pointed at + * `postgres://everythingdev:everythingdev@localhost:/` on the + * hardcoded 5432–5435 block. Those are exactly the opt-ins that made two + * concurrent local projects fight over host ports. + * + * The compose-era convention is retired — local persistence is a + * file-backed PGlite data dir (`.bos/db`), tests run on in-memory pglite. + * `bos dev` repairs leftover state once, idempotently: + * 1. `.env` lines whose URL matches the conventional shape (user, host, + * and database name — port too, but drift ports like 5532 from the old + * +100 allocator also match by name) are rewritten to the new defaults. + * Any other `postgres://` URL is operator-owned and left alone. + * 2. A leftover scaffold-shaped `docker-compose.yml` is moved to + * `.bos/compose-retired/` rather than deleted; operator-authored + * compose files (different service names/images) are untouched. + */ +export const migrateLegacyComposeState = ( + configDir: string, +): Effect.Effect => + Effect.try({ + try: () => { + const envPath = join(configDir, ".env"); + const databaseUrlDrift: LegacyMigrationReport["databaseUrlDrift"] = []; + + if (existsSync(envPath)) { + const raw = readFileSync(envPath, "utf-8"); + const parsed = loadDotenv({ path: envPath, quiet: true }).parsed ?? {}; + const rewritten = rewrittenLegacyDatabaseValues({ file: parsed, raw }); + if (rewritten.size > 0) { + const lines = raw.split("\n"); + const updated = lines.map((line) => { + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/.exec(line); + if (!match) return line; + const replacement = rewritten.get(match[1]); + return replacement === undefined ? line : `${match[1]}=${replacement}`; + }); + writeFileSync(envPath, updated.join("\n")); + for (const [key, to] of rewritten) { + databaseUrlDrift.push({ key, from: parsed[key] ?? "", to }); + } + } + } + + const composeRelocated = relocateLegacyComposeFile(configDir); + return { databaseUrlDrift, composeRelocated }; + }, + catch: (cause) => new LegacyMigrationError({ cause }), + }).pipe(Effect.orElseSucceed(() => ({ databaseUrlDrift: [], composeRelocated: false }))); + +/** Matches the compose-era URL shapes: the scaffolded user + localhost + the convention's database names. */ +const LEGACY_DATABASE_NAME = /^(api_db|auth_db|api_test_db|auth_test_db)$/; + +function legacyMappingForUrl(rawUrl: string): string | undefined { + try { + const url = new URL(rawUrl); + if (url.protocol !== "postgres:" && url.protocol !== "postgresql:") return undefined; + if (url.hostname !== "localhost" && url.hostname !== "127.0.0.1") return undefined; + if (url.username !== "everythingdev" || url.password !== "everythingdev") return undefined; + const databaseName = decodeURIComponent(url.pathname.replace(/^\//, "")); + if (!LEGACY_DATABASE_NAME.test(databaseName)) return undefined; + return databaseName.endsWith("_test_db") ? PGLITE_TEST_DATABASE_URL : PGLITE_DEV_DATABASE_URL; + } catch { + return undefined; + } +} + +export function rewrittenLegacyDatabaseValues(input: { + file: Record; + raw: string; +}): Map { + const rewritten = new Map(); + for (const [key, value] of Object.entries(input.file)) { + const match = value ? legacyMappingForUrl(value) : undefined; + if (match !== undefined) { + const rawLine = new RegExp(`^${key}=.+$`, "m").test(input.raw); + if (rawLine) rewritten.set(key, match); + } + } + return rewritten; +} + +const SCAFFOLD_SERVICE_NAMES = new Set([ + "postgres-api", + "postgres-auth", + "postgres-api-test", + "postgres-auth-test", +]); +const SCAFFOLD_IMAGE_HINTS = [ + "public.ecr.aws/docker/library/postgres", + "docker.io/library/postgres", + "postgres:17-alpine", + "ghcr.io/nearbuilders/mirror-postgres", +]; + +function isScaffoldComposeFile(raw: string): boolean { + const services = [...raw.matchAll(/^\s+-?\s*([a-z-]+):\s*$/gm)].map((match) => match[1] ?? ""); + if (!services.some((name) => SCAFFOLD_SERVICE_NAMES.has(name))) return false; + return SCAFFOLD_IMAGE_HINTS.some((hint) => raw.includes(hint)); +} + +function relocateLegacyComposeFile(configDir: string): boolean { + const composePath = join(configDir, "docker-compose.yml"); + if (!existsSync(composePath)) return false; + let raw: string; + try { + raw = readFileSync(composePath, "utf-8"); + } catch { + return false; + } + if (!isScaffoldComposeFile(raw)) return false; + const retiredDir = join(configDir, ".bos", "compose-retired"); + mkdirSync(retiredDir, { recursive: true }); + renameSync(composePath, join(retiredDir, "docker-compose.yml")); + return true; +} diff --git a/packages/everything-dev/src/infra/docker.ts b/packages/everything-dev/src/infra/docker.ts deleted file mode 100644 index 3b66a0c19..000000000 --- a/packages/everything-dev/src/infra/docker.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { existsSync } from "node:fs"; -import { join } from "node:path"; -import process from "node:process"; -import { execa } from "execa"; -import { stripAnsi } from "../services/dev/logs"; -import type { PreflightFailure } from "./preflight"; - -export interface DockerAutostartGuards { - composeFileExists: boolean; - dockerAvailable: boolean; - testMode: boolean; -} - -export interface DockerComposeResult { - ok: boolean; - tail: string; -} - -const TAIL_LINES = 15; - -export function takeTail(lines: string[], count: number = TAIL_LINES): string { - return lines.slice(-count).join("\n").trim(); -} - -export function composeFilePath(configDir: string): string { - return join(configDir, "docker-compose.yml"); -} - -export function isDockerTestMode(env: Record): boolean { - return env.BOS_TEST === "1" || env.NODE_ENV === "test" || env.BOS_NO_PERSIST_PORTS === "1"; -} - -export function shouldAutoStartDocker( - failures: PreflightFailure[], - guards: DockerAutostartGuards, -): boolean { - if (failures.length === 0) return false; - if (guards.testMode || !guards.dockerAvailable || !guards.composeFileExists) return false; - return failures.every((failure) => failure.tcpReachable === false); -} - -export async function isDockerAvailable(): Promise { - try { - const result = await execa("docker", ["info"], { stdio: "pipe", reject: false, timeout: 5000 }); - return (result.exitCode ?? 1) === 0; - } catch { - return false; - } -} - -export async function runDockerComposeUp(configDir: string): Promise { - try { - const result = await execa("docker", ["compose", "up", "-d", "--wait"], { - cwd: configDir, - stdio: "pipe", - reject: false, - timeout: 5 * 60_000, - }); - const lines = [result.stdout ?? "", result.stderr ?? ""] - .flatMap((output) => output.split("\n")) - .map((line) => stripAnsi(line).trim()) - .filter((line) => line.length > 0); - return { ok: (result.exitCode ?? 1) === 0, tail: takeTail(lines) }; - } catch (error) { - return { ok: false, tail: error instanceof Error ? error.message : String(error) }; - } -} - -export function detectAutoStartContext(configDir: string): DockerAutostartGuards { - return { - composeFileExists: existsSync(composeFilePath(configDir)), - dockerAvailable: false, - testMode: isDockerTestMode(process.env as Record), - }; -} diff --git a/packages/everything-dev/src/infra/materializer.ts b/packages/everything-dev/src/infra/materializer.ts index 9e27d4df6..90e1b1ef8 100644 --- a/packages/everything-dev/src/infra/materializer.ts +++ b/packages/everything-dev/src/infra/materializer.ts @@ -106,8 +106,8 @@ export interface MaterializeOptions { /** * Orchestration helper: materializes the env templates in the canonical - * order. docker-compose.yml is a static committed file — nothing here - * generates or rewrites it. + * order. Local databases come from each workspace's persistent `.bos/db` + * PGlite default — nothing here provisions a server. */ export async function materializeViaLayer( configDir: string, diff --git a/packages/everything-dev/src/infra/planner.ts b/packages/everything-dev/src/infra/planner.ts index 4d7e6e102..3b65a8228 100644 --- a/packages/everything-dev/src/infra/planner.ts +++ b/packages/everything-dev/src/infra/planner.ts @@ -6,13 +6,7 @@ import { buildDescription, isAuthMirrorPluginEntry, } from "../services/dev/service-descriptor"; -import { - buildConventionalDatabases, - buildConventionalRedis, - getSecretGroups, - loadPortState, - savePortState, -} from "../services/infra"; +import { loadPortState, savePortState } from "../services/infra"; import type { RuntimeConfig } from "../types"; import { shouldPersistPortState } from "./materializer"; import { ownerOfPort } from "./port-ownership"; @@ -20,10 +14,8 @@ import { PortAllocator, type PortBlockEntry } from "./ports"; import type { ClaimRecord, CliPorts, - DatabasePlan, InfraInput, InfraPlan, - RedisPlan, ResolvedPorts, RuntimeLaunchSpec, ServiceDescriptorPlan, @@ -32,9 +24,6 @@ import { InfraError } from "./types"; const DEFAULT_HOST_PORT = 3000; -const POSTGRES_USER = "everythingdev"; -const POSTGRES_PASSWORD = "everythingdev"; - export function workspaceKey(configDir: string): string { const hash = createHash("sha256").update(resolve(configDir)).digest("hex"); return hash.slice(0, 12); @@ -241,87 +230,6 @@ function allocateServices( ); } -function allocateDatabases( - runtimeConfig: RuntimeConfig, - operatorEnv: Record, -): Effect.Effect< - { - dbs: DatabasePlan[]; - redisPlans: RedisPlan[]; - }, - InfraError, - PortAllocator -> { - return Effect.gen(function* () { - const groups = getSecretGroups(runtimeConfig); - const allSecrets = groups.flatMap((group) => group.secrets); - const allocator = yield* PortAllocator; - - // Zero-config dev (plan 055 ticket 11): database/redis URLs are opt-in. - // When the operator supplies none (shell or .env), nothing conventional is - // generated — consumers fall back to their own boot-schema defaults - // (ephemeral pglite) and the compose preflight has nothing to probe, so it - // is skipped entirely. A partial supply generates only the missing - // conventional URLs (compose serves both databases, same as before). - const operatorSubstrateKeys = Object.keys(operatorEnv).filter( - (key) => - (key.endsWith("_DATABASE_URL") || key.endsWith("_REDIS_URL")) && operatorEnv[key] !== "", - ); - if (operatorSubstrateKeys.length === 0) { - return { dbs: [], redisPlans: [] }; - } - - const pendingDatabases = buildConventionalDatabases(allSecrets).filter( - (db) => !operatorEnv[db.secret], - ); - const pendingRedis = buildConventionalRedis(allSecrets).filter((r) => !operatorEnv[r.secret]); - if (pendingDatabases.length === 0 && pendingRedis.length === 0) { - return { dbs: [], redisPlans: [] }; - } - - const dbs: DatabasePlan[] = []; - const allocatedByDesired = new Map(); - for (const db of pendingDatabases) { - let port: number; - if (allocatedByDesired.has(db.port)) { - port = allocatedByDesired.get(db.port)!; - } else { - port = yield* allocator.pickAvailable(db.port); - allocatedByDesired.set(db.port, port); - } - dbs.push({ - secret: db.secret, - slug: db.slug, - port, - dbName: db.databaseName, - url: `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${port}/${db.databaseName}`, - }); - } - - const redisPlans: RedisPlan[] = []; - for (const r of pendingRedis) { - const port = yield* allocator.pickAvailable(r.port); - redisPlans.push({ - secret: r.secret, - slug: r.slug, - port, - url: `redis://localhost:${port}`, - }); - } - - return { dbs, redisPlans }; - }).pipe( - Effect.mapError( - (portErr) => - new InfraError({ - phase: "allocate-databases", - message: `Database port allocation failed: ${String(portErr)}`, - cause: portErr, - }), - ), - ); -} - export function buildServiceDescriptors( runtimeConfig: RuntimeConfig, resolvedPorts: ResolvedPorts, @@ -432,11 +340,7 @@ export function buildLaunchSpec( }; } -export function buildEnvGenerated( - resolvedPorts: ResolvedPorts, - dbs: DatabasePlan[], - redisPlans: RedisPlan[], -): Record { +export function buildEnvGenerated(resolvedPorts: ResolvedPorts): Record { const env: Record = {}; if (resolvedPorts.host) { @@ -446,13 +350,6 @@ export function buildEnvGenerated( env.BASE_URL = `http://localhost:${resolvedPorts.host}`; } - for (const db of dbs) { - env[db.secret] = db.url; - } - for (const r of redisPlans) { - env[r.secret] = r.url; - } - return env; } @@ -477,23 +374,8 @@ export function planInfra(input: InfraInput): Effect.Effect p?.source === "local" || p?.ui?.source === "local"); - - const { dbs, redisPlans } = hasLocalServices - ? yield* allocateDatabases(input.bosConfig, input.operatorEnv ?? {}) - : { dbs: [], redisPlans: [] }; - + // DB/Redis port conventions are gone (PGlite is the only local database — + // no ports); nothing to allocate or probe for substrate URLs. const resolvedPorts: ResolvedPorts = { ...svcPorts }; // Write merged state once after all allocations succeed @@ -590,7 +472,7 @@ export function planInfra(input: InfraInput): Effect.Effect d.key); const descriptionMap = new Map(serviceDescriptors.map((d) => [d.key, d])); diff --git a/packages/everything-dev/src/infra/preflight.ts b/packages/everything-dev/src/infra/preflight.ts index 83e03597f..2a391a244 100644 --- a/packages/everything-dev/src/infra/preflight.ts +++ b/packages/everything-dev/src/infra/preflight.ts @@ -27,7 +27,7 @@ export interface PreflightFailure { host: string; port: number; error: string; - /** True when the target port accepted a TCP connection (so docker compose cannot fix this failure). */ + /** True when the target port accepted a TCP connection (so this is not a missing local server). */ tcpReachable: boolean; } @@ -143,8 +143,8 @@ export function preflightLocalInfra( const appDb = target.secret === "API_DATABASE_URL" || target.secret === "AUTH_DATABASE_URL"; const pluginContext = appDb - ? " Run `docker compose up -d --wait` to start local Postgres." - : ` The plugin for ${target.secret} runs inside the local host process, so this DB must be reachable. Run \`docker compose up -d --wait\` to start local Postgres.`; + ? " Point it at a postgres:// URL you control (local persistence needs no server — clear the URL to fall back to the persistent .bos/db PGlite)." + : ` The plugin for ${target.secret} runs inside the local host process, so this DB must be reachable. Point it at a postgres:// URL you control or clear the URL to fall back to the persistent .bos/db PGlite.`; if (tcpOk) { return { secret: target.secret, diff --git a/packages/everything-dev/src/infra/types.ts b/packages/everything-dev/src/infra/types.ts index b6a43a9bb..8ecb288a8 100644 --- a/packages/everything-dev/src/infra/types.ts +++ b/packages/everything-dev/src/infra/types.ts @@ -57,25 +57,9 @@ export interface ServiceDescriptorPlan { localPath?: string; } -export interface DatabasePlan { - secret: string; - slug: string; - port: number; - dbName: string; - url: string; -} - -export interface RedisPlan { - secret: string; - slug: string; - port: number; - url: string; -} - export type InfraPhase = | "resolve-config" | "allocate-services" - | "allocate-databases" | "claim" | "materialize-env" | "launch"; diff --git a/packages/everything-dev/src/services/codegen/api-contract.ts b/packages/everything-dev/src/services/codegen/api-contract.ts index 84a930ae0..157c68f39 100644 --- a/packages/everything-dev/src/services/codegen/api-contract.ts +++ b/packages/everything-dev/src/services/codegen/api-contract.ts @@ -722,7 +722,7 @@ export function writeGeneratedFiles(opts: { continue; for (const genName of ["plugins-client.gen.ts", "auth-types.gen.ts"]) { const genPath = join(pluginsRoot, entry.name, "src", "lib", genName); - if (existsSync(genPath)) rmSync(genPath); + if (existsSync(genPath)) rmSync(genPath, { force: true }); } } } diff --git a/packages/everything-dev/src/services/deploy/smoke.ts b/packages/everything-dev/src/services/deploy/smoke.ts index 82d54f177..7050dd944 100644 --- a/packages/everything-dev/src/services/deploy/smoke.ts +++ b/packages/everything-dev/src/services/deploy/smoke.ts @@ -214,14 +214,12 @@ export async function runSmoke(options: SmokeOptions): Promise { const envPairs: Record = { BOS_ACCOUNT: options.account ?? process.env.SMOKE_BOS_ACCOUNT ?? "dev.everything.near", BOS_GATEWAY: options.gateway ?? process.env.SMOKE_BOS_GATEWAY ?? "everything.dev", + // Boot-gate only: the container's own persistent-by-path PGlite default + // (in-container fs) is enough — no external postgres involved. API_DATABASE_URL: - options.apiDatabaseUrl ?? - process.env.SMOKE_API_DATABASE_URL ?? - "postgres://everythingdev:everythingdev@host.docker.internal:5434/api_test_db", + options.apiDatabaseUrl ?? process.env.SMOKE_API_DATABASE_URL ?? "pglite:.bos/db", AUTH_DATABASE_URL: - options.authDatabaseUrl ?? - process.env.SMOKE_AUTH_DATABASE_URL ?? - "postgres://everythingdev:everythingdev@host.docker.internal:5435/auth_test_db", + options.authDatabaseUrl ?? process.env.SMOKE_AUTH_DATABASE_URL ?? "pglite:.bos/db", BETTER_AUTH_SECRET: options.authSecret ?? process.env.SMOKE_BETTER_AUTH_SECRET ?? @@ -235,7 +233,6 @@ export async function runSmoke(options: SmokeOptions): Promise { container, "-p", `${port}:${port}`, - "--add-host=host.docker.internal:host-gateway", ...Object.entries(envPairs).flatMap(([key, value]) => ["-e", `${key}=${value}`]), image, ]); diff --git a/packages/everything-dev/src/services/dev/boot.ts b/packages/everything-dev/src/services/dev/boot.ts index f83752ff5..06f28e5d4 100644 --- a/packages/everything-dev/src/services/dev/boot.ts +++ b/packages/everything-dev/src/services/dev/boot.ts @@ -1,6 +1,7 @@ import { randomBytes } from "node:crypto"; import { Data, Effect } from "effect"; import type { DevOptions, PhaseTiming } from "../../contract"; +import { migrateLegacyComposeState } from "../../env/legacy-databases"; import { getProcessEnv, operatorEnvRecord, setProcessEnv } from "../../env/process-env"; import { captureShellEnv, @@ -8,12 +9,6 @@ import { ProjectEnv, type ProjectEnvService, } from "../../env/project-env"; -import { - detectAutoStartContext, - isDockerAvailable, - runDockerComposeUp, - shouldAutoStartDocker, -} from "../../infra/docker"; import { materializeViaLayer } from "../../infra/materializer"; import { planInfra } from "../../infra/planner"; import { preflightLocalInfra } from "../../infra/preflight"; @@ -248,6 +243,18 @@ export const devBootstrap = (deps: BootstrapDeps, input: DevOptions, timings: Ph ), ); + const legacy = yield* migrateLegacyComposeState(session.root); + for (const { key, to } of legacy.databaseUrlDrift) { + yield* Effect.logInfo( + `[env] ${key} migrated from the retired compose-era Postgres convention → ${to}`, + ); + } + if (legacy.composeRelocated) { + yield* Effect.logInfo( + `[env] scaffold-era docker-compose.yml relocated to .bos/compose-retired — local databases are the persistent .bos/db PGlite now`, + ); + } + const needsEphemeralSecret = !("BETTER_AUTH_SECRET" in operatorEnv) || operatorEnv.BETTER_AUTH_SECRET === ""; const mergedEnv = yield* Effect.sync(() => { @@ -266,32 +273,7 @@ export const devBootstrap = (deps: BootstrapDeps, input: DevOptions, timings: Ph if (key === "BASE_URL" || key === "CORS_ORIGIN") setProcessEnv(key, value); } }); - let preflightFailures = yield* preflightLocalInfra(plan.envGenerated, mergedEnv); - if ( - preflightFailures.length > 0 && - shouldAutoStartDocker(preflightFailures, { - ...detectAutoStartContext(session.root), - dockerAvailable: yield* Effect.promise(isDockerAvailable), - }) - ) { - const compose = yield* step(timings, "docker compose up", () => - runDockerComposeUp(session.root), - ); - if (compose.ok) { - preflightFailures = yield* preflightLocalInfra(plan.envGenerated, mergedEnv); - } else { - preflightFailures = [ - ...preflightFailures, - { - secret: "docker-compose", - host: "localhost", - port: 0, - error: `docker compose up -d --wait failed${compose.tail ? `:\n${compose.tail}` : ""}`, - tcpReachable: false, - }, - ]; - } - } + const preflightFailures = yield* preflightLocalInfra(plan.envGenerated, mergedEnv); if (preflightFailures.length > 0) { return yield* new DevPreflightFailed({ messages: preflightFailures.map((f) => f.error) }); } diff --git a/packages/everything-dev/src/services/infra.ts b/packages/everything-dev/src/services/infra.ts index eac7346ae..036bc838f 100644 --- a/packages/everything-dev/src/services/infra.ts +++ b/packages/everything-dev/src/services/infra.ts @@ -4,9 +4,6 @@ import { dirname, join } from "node:path"; import type { RuntimeConfig } from "../types"; import { frameworkOwnedHeaderComment } from "./scaffold/ownership"; -const POSTGRES_USER = "everythingdev"; -const POSTGRES_PASSWORD = "everythingdev"; -const AUTH_DATABASE_SECRET = "AUTH_DATABASE_URL"; const HOST_SECRET = "CORS_ORIGIN"; const REDIS_PORT = 6379; const TEST_AUTH_SECRET = "regression-test-secret-do-not-use-in-production"; @@ -16,21 +13,18 @@ const VESTIGIAL_SECRETS = new Set([ ]); /** - * Fixed local-development database conventions. The committed - * docker-compose.yml provisions exactly these services/ports — nothing is - * derived from runtime config. Every non-auth `*_DATABASE_URL` secret shares - * the api database (schema-isolated per plugin); auth gets its own. + * Local development conventions. Databases default to a persistent, + * file-backed PGlite data dir per workspace process (`.bos/db`) — no ports, + * no docker. Operators point a secret at a `postgres://` URL to demand a + * shared server. The two-database split (auth separate from everything + * else) survives independent of compose. */ -const DEV_API_DB_PORT = 5432; -const DEV_AUTH_DB_PORT = 5433; -const TEST_API_DB_PORT = 5434; -const TEST_AUTH_DB_PORT = 5435; +export const PGLITE_DEV_DATABASE_URL = "pglite:.bos/db"; +export const PGLITE_TEST_DATABASE_URL = "pglite::memory:"; export interface DatabaseConvention { secret: string; slug: string; - port: number; - databaseName: string; url: string; } @@ -99,8 +93,8 @@ function normalizeRedisSlug(secret: string): string { return secret.replace(/_REDIS_URL$/, "").toLowerCase(); } -function databaseUrl(port: number, databaseName: string): string { - return `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:${port}/${databaseName}`; +function databaseUrl(): string { + return PGLITE_DEV_DATABASE_URL; } export function buildConventionalDatabases(secrets: string[]): DatabaseConvention[] { @@ -112,10 +106,7 @@ export function buildConventionalDatabases(secrets: string[]): DatabaseConventio .sort((a, b) => normalizeDatabaseSlug(a).localeCompare(normalizeDatabaseSlug(b))) .map((secret) => { const slug = normalizeDatabaseSlug(secret); - const isAuth = secret === AUTH_DATABASE_SECRET; - const port = isAuth ? DEV_AUTH_DB_PORT : DEV_API_DB_PORT; - const databaseName = isAuth ? "auth_db" : "api_db"; - return { secret, slug, port, databaseName, url: databaseUrl(port, databaseName) }; + return { secret, slug, url: databaseUrl() }; }); } @@ -132,11 +123,8 @@ export function buildConventionalRedis(secrets: string[]): RedisConvention[] { })); } -function testDatabaseUrl(secret: string): string { - const isAuth = secret === AUTH_DATABASE_SECRET; - const port = isAuth ? TEST_AUTH_DB_PORT : TEST_API_DB_PORT; - const databaseName = isAuth ? "auth_test_db" : "api_test_db"; - return databaseUrl(port, databaseName); +function testDatabaseUrl(): string { + return PGLITE_TEST_DATABASE_URL; } export function getSecretGroups(runtimeConfig: RuntimeConfig): SecretGroup[] { @@ -195,8 +183,9 @@ function defaultSecretValue( } // Zero-config dev (plan 055 ticket 11): substrate URLs are opt-in — the - // example renders them empty so the consumers' pglite boot defaults apply; - // the operator sets a postgres:// URL to demand compose Postgres. + // example renders them empty so the consumers' persistent pglite boot + // defaults apply; the operator sets a postgres:// URL for production or + // shared access. if (options.forExample) { return ""; } @@ -258,7 +247,7 @@ function renderEnvFile( lines.push( secret.endsWith("_REDIS_URL") ? "# Leave empty for dev defaults; set a redis:// URL to opt into a real Redis." - : "# Leave empty to run dev on ephemeral pglite; set a postgres:// URL to opt into compose Postgres.", + : "# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access.", ); } lines.push(`${secret}=${defaultSecretValue(secret, databaseMap, redisMap, options)}`); @@ -274,7 +263,7 @@ export { renderEnvFile }; function renderEnvTestFile(groups: SecretGroup[]): string { const lines: string[] = [ "# Generated test environment — loaded by test suites instead of .env", - "# Test databases run on dedicated *-test docker-compose services", + "# Test databases run on in-memory pglite — no docker involved", "", ]; @@ -284,7 +273,7 @@ function renderEnvTestFile(groups: SecretGroup[]): string { if (secret === "BETTER_AUTH_SECRET") { entries.push(`${secret}=${TEST_AUTH_SECRET}`); } else if (secret.endsWith("_DATABASE_URL")) { - entries.push(`${secret}=${testDatabaseUrl(secret)}`); + entries.push(`${secret}=${testDatabaseUrl()}`); } } if (entries.length > 0) { diff --git a/packages/everything-dev/src/services/resolution/serialize.ts b/packages/everything-dev/src/services/resolution/serialize.ts index 4f2e520a8..3bf9de972 100644 --- a/packages/everything-dev/src/services/resolution/serialize.ts +++ b/packages/everything-dev/src/services/resolution/serialize.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { type BosConfig, BosConfigSchema } from "../../types"; import { readLocalAuthoredConfigInput } from "./load"; @@ -54,7 +54,13 @@ export function writeResolvedConfig( } catch { // file doesn't exist yet } - writeFileSync(resolvedPath, content); + // Atomic rename, not a plain write: concurrent dev sessions in one config + // dir are a supported flow, and a plain write can hand the other session a + // half-written file (torn read) — a missing plugins.localPath then flips + // the contract-bridge sweep into deleting that plugin's fresh gen files. + const tempPath = `${resolvedPath}.${process.pid}.tmp`; + writeFileSync(tempPath, content); + renameSync(tempPath, resolvedPath); } export function resolveBosConfigPath(configDir: string): string { diff --git a/packages/everything-dev/src/services/scaffold/init-flow.ts b/packages/everything-dev/src/services/scaffold/init-flow.ts index 19630cc35..bf58048ad 100644 --- a/packages/everything-dev/src/services/scaffold/init-flow.ts +++ b/packages/everything-dev/src/services/scaffold/init-flow.ts @@ -371,7 +371,7 @@ export class InitFlowService extends Context.Service< ); const initRuntime = initConfig?.runtime; if (initRuntime) { - await timePhase(timings, "generate env/docker", async () => { + await timePhase(timings, "generate env", async () => { await materializeViaLayer(targetDir, initRuntime); }); } diff --git a/packages/everything-dev/src/services/scaffold/init.ts b/packages/everything-dev/src/services/scaffold/init.ts index 60031f678..078a745ee 100644 --- a/packages/everything-dev/src/services/scaffold/init.ts +++ b/packages/everything-dev/src/services/scaffold/init.ts @@ -184,7 +184,6 @@ export function buildInitPatterns( if (has("ui")) patterns.push("ui/**"); if (has("api")) patterns.push(API_TEMPLATE_PATTERN); - if (has("api") || has("host")) patterns.push(COMPOSE_TEMPLATE_PATTERN); if (has("host")) patterns.push("host/**"); if (has("plugins")) { for (const plugin of plugins ?? []) { @@ -199,17 +198,10 @@ export function buildInitPatterns( /** api-override children get the slim generic shell, never the parent's domain API. */ const API_TEMPLATE_PATTERN = ".github/templates/api/**"; -/** Child-sized compose (api + api-test databases) for local compute overrides. */ -const COMPOSE_TEMPLATE_PATTERN = ".github/templates/docker-compose.yml"; - export function isApiTemplatePath(filePath: string): boolean { return filePath.startsWith(".github/templates/api/"); } -export function isComposeTemplatePath(filePath: string): boolean { - return filePath === COMPOSE_TEMPLATE_PATTERN; -} - export function buildPluginRouteExclusions( parentConfig: { plugins?: Record } | null | undefined, selectedPlugins: string[], @@ -303,9 +295,6 @@ export function sourcePathToDestinationPath(filePath: string): string { if (isApiTemplatePath(filePath)) { return filePath.replace(/^\.github\/templates\/api\//, "api/"); } - if (isComposeTemplatePath(filePath)) { - return "docker-compose.yml"; - } return filePath.startsWith(".github/templates/") ? filePath.replace(/^\.github\/templates\//, ".github/") : filePath; @@ -372,11 +361,6 @@ export async function copyFilteredFiles( if (isApiTemplatePath(match)) allFiles.delete(match); } } - if (!options.overrides.includes("api") && !options.overrides.includes("host")) { - for (const match of allFiles) { - if (isComposeTemplatePath(match)) allFiles.delete(match); - } - } mkdirSync(destination, { recursive: true }); @@ -566,12 +550,6 @@ export function buildChildRootScripts(sections: { // Scripts key off the child's own override selection (what it runs locally), // not off resolved-config secrets — a ui-only child gets none of these. - if (sections.api || sections.host) { - scripts["dev:postgres"] = "docker compose up -d --wait && pnpm run dev"; - scripts["dev:postgres:down"] = "docker compose down"; - scripts["dev:postgres:reset"] = "docker compose down -v && docker compose up -d --wait"; - } - if (sections.ui) { scripts["dev:ui"] = "bos dev --ui local --api remote"; } @@ -949,9 +927,6 @@ export async function personalizeConfig( "test:api", "test:integration", "test:e2e", - "dev:postgres", - "dev:postgres:down", - "dev:postgres:reset", "dev:ui", "dev:api", ]) { @@ -1385,11 +1360,6 @@ export async function writeInitSnapshot( if (isApiTemplatePath(match)) allFiles.delete(match); } } - if (!options.overrides.includes("api") && !options.overrides.includes("host")) { - for (const match of allFiles) { - if (isComposeTemplatePath(match)) allFiles.delete(match); - } - } const fileHashes: Record = {}; for (const filePath of allFiles) { @@ -1439,7 +1409,6 @@ export async function generateDatabaseMigrations(destination: string): Promise = { pnpm: 5 * 60_000, - docker: 5 * 60_000, node_modules: 2 * 60_000, tar: 60_000, }; @@ -1551,7 +1520,7 @@ You don't need to wait for a PR to merge and run through CI/CD. Publish your own ### Starting Development 1. \`pnpm install\` 2. \`pnpm run dev\` -3. \`bos dev\` creates \`.env\` on first run and starts local Postgres via docker compose when it is down`); +3. \`bos dev\` creates \`.env\` on first run; local data is a persistent PGlite data dir (\`.bos/db\`), no docker involved`); parts.push(`### Debugging Issues @@ -1776,7 +1745,6 @@ function generateGitignore(): string { dist/ .env .bos/ -docker-compose.yml *.gen.ts *.gen.tsx `; diff --git a/packages/everything-dev/src/services/start/boot.ts b/packages/everything-dev/src/services/start/boot.ts index a4503bc50..19f6d2003 100644 --- a/packages/everything-dev/src/services/start/boot.ts +++ b/packages/everything-dev/src/services/start/boot.ts @@ -226,8 +226,9 @@ export const startBootstrap = (deps: BootstrapDeps, input: StartOptions) => if (!operatorEnvRecord().API_DATABASE_URL && !operatorEnvRecord().AUTH_DATABASE_URL) { warnings.push( - "No *_DATABASE_URL set — plugins without explicit URLs boot on ephemeral pglite and their " + - "data will not survive restarts. Set API_DATABASE_URL/AUTH_DATABASE_URL in .env for production.", + "No *_DATABASE_URL set — plugins without explicit URLs boot on the local persistent PGlite " + + "default (.bos/db, per process). Set API_DATABASE_URL/AUTH_DATABASE_URL in .env to a " + + "postgres:// URL for production durability.", ); } diff --git a/packages/everything-dev/src/utils/timing.ts b/packages/everything-dev/src/utils/timing.ts index 26cc11294..3c660c8dd 100644 --- a/packages/everything-dev/src/utils/timing.ts +++ b/packages/everything-dev/src/utils/timing.ts @@ -30,13 +30,12 @@ const PHASE_LABELS: Record = { "personalize config": "Personalizing config...", "write snapshot": "Writing snapshot...", "resolve config": "Resolving config...", - "generate env/docker": "Generating environment config...", + "generate env": "Generating environment config...", "create env file": "Creating .env file...", "install dependencies": "Installing dependencies...", "generate types": "Generating types...", "generate migrations": "Generating database migrations...", "generate code artifacts": "Generating code artifacts...", - "docker compose up": "Starting Docker services...", }; function phaseLabel(name: string): string { diff --git a/packages/everything-dev/tests/integration/init.compose.test.ts b/packages/everything-dev/tests/integration/init.compose.test.ts index 084edadeb..fa2ac26dc 100644 --- a/packages/everything-dev/tests/integration/init.compose.test.ts +++ b/packages/everything-dev/tests/integration/init.compose.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -6,6 +6,11 @@ import { buildInitPatterns, copyFilteredFiles } from "../../src/cli/init"; const REPO_ROOT = join(import.meta.dirname, "../../../../"); +/** + * ADR 0038: local databases are the persistent PGlite data dirs — the + * compose-era infra delivery is retired. `bos init` must never emit a + * docker-compose.yml for any child shape. + */ async function scaffold(overrides: Array<"ui" | "api" | "host" | "plugins">): Promise { const projectDir = mkdtempSync(join(tmpdir(), "bos-init-compose-")); const patterns = buildInitPatterns(overrides, overrides.includes("plugins") ? ["template"] : []); @@ -18,7 +23,7 @@ async function scaffold(overrides: Array<"ui" | "api" | "host" | "plugins">): Pr return projectDir; } -describe("init docker-compose delivery", () => { +describe("init never delivers docker-compose (ADR 0038)", () => { const tempDirs: string[] = []; afterEach(() => { @@ -30,21 +35,19 @@ describe("init docker-compose delivery", () => { } }); - it("api-override children get the 2-service child compose", async () => { + it("api-override children get no compose", async () => { const projectDir = await scaffold(["ui", "api"]); tempDirs.push(projectDir); - const compose = readFileSync(join(projectDir, "docker-compose.yml"), "utf-8"); - expect(compose).toContain("postgres-api:"); - expect(compose).toContain("postgres-api-test:"); - expect(compose).not.toContain("postgres-auth"); + expect(existsSync(join(projectDir, "docker-compose.yml"))).toBe(false); }); - it("children overriding host get the compose too", async () => { + it("children overriding host get no compose either", async () => { const projectDir = await scaffold(["host"]); tempDirs.push(projectDir); - expect(existsSync(join(projectDir, "docker-compose.yml"))).toBe(true); + expect(existsSync(join(projectDir, "docker-compose.yml"))).toBe(false); + expect(existsSync(join(projectDir, ".github", "docker-compose.yml"))).toBe(false); }); it("children without local compute get no compose", async () => { diff --git a/packages/everything-dev/tests/integration/init.relative-directory.test.ts b/packages/everything-dev/tests/integration/init.relative-directory.test.ts index c115a8b54..cecddec7b 100644 --- a/packages/everything-dev/tests/integration/init.relative-directory.test.ts +++ b/packages/everything-dev/tests/integration/init.relative-directory.test.ts @@ -91,24 +91,18 @@ describe("bos init - relative directory", () => { expect(existsSync(join(targetDir, "bos.config.json"))).toBe(true); expect(existsSync(join(targetDir, ".env.example"))).toBe(true); - expect(existsSync(join(targetDir, "docker-compose.yml"))).toBe(true); + expect(existsSync(join(targetDir, "docker-compose.yml"))).toBe(false); const envExample = readFileSync(join(targetDir, ".env.example"), "utf-8"); - const dockerCompose = readFileSync(join(targetDir, "docker-compose.yml"), "utf-8"); // Auth env materializes when the auth workspace is present in the child // (composed inits); the ui/api-only scaffold carries the api + host vars. // Substrate URLs render empty with an opt-in hint (plan 055 ticket 11). expect(envExample).toContain("API_DATABASE_URL="); expect(envExample).toContain( - "# Leave empty to run dev on ephemeral pglite; set a postgres:// URL to opt into compose Postgres.", + "# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access.", ); expect(envExample).toContain("CORS_ORIGIN=http://localhost:3000"); expect(envExample).not.toContain("PROJECTS_DATABASE_URL="); - - expect(dockerCompose).toContain("postgres-api:"); - expect(dockerCompose).toContain("postgres-api-test:"); - expect(dockerCompose).not.toContain("postgres-auth:"); - expect(dockerCompose).not.toContain("postgres-example:"); }, 60_000); }); diff --git a/packages/everything-dev/tests/unit/codegen-artifacts-concurrency.test.ts b/packages/everything-dev/tests/unit/codegen-artifacts-concurrency.test.ts new file mode 100644 index 000000000..3ef90cf31 --- /dev/null +++ b/packages/everything-dev/tests/unit/codegen-artifacts-concurrency.test.ts @@ -0,0 +1,165 @@ +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { syncApiContractBridge } from "../../src/services/codegen/api-contract"; +import { writeResolvedConfig } from "../../src/services/resolution/serialize"; +import type { BosConfig, RuntimeConfig } from "../../src/types"; + +/** + * Concurrent `bos dev` sessions in one config dir are a supported flow, and + * their artifact generations raced: a torn `bos.resolved-config.json` read + * resolved plugins without a localPath, flipping the sweep into deleting + * another boot's freshly written gen files, and the sweep's unguarded + * `rmSync` crashed with ENOENT. The resolved-config write is atomic (tmp + + * rename) and the sweep is force-tolerant — this test exercises the real + * bridge under concurrency so neither flavor can regress silently. + */ + +const CONFIG = { + account: "race.near", + domain: "race.dev", + app: { + host: { + name: "host", + url: "http://localhost:3000", + entry: "./src/program.ts", + source: "local", + }, + ui: { name: "ui", url: "http://localhost:3003", entry: "./src/entry.tsx", source: "local" }, + api: { name: "api", url: "http://localhost:3001", entry: "./src/index.ts", source: "local" }, + }, +} as unknown as BosConfig; + +const PLUGIN_KEYS = ["fakea", "fakeb"]; + +function writeContractSource(dir: string, name: string): void { + mkdirSync(join(dir, "src"), { recursive: true }); + writeFileSync( + join(dir, "src", "contract.ts"), + `import { oc } from "@orpc/contract"; +import { z } from "zod"; + +export const ${name}Contract = { + ping: oc.route({ method: "GET", path: "/ping" }).output(z.object({ ok: z.boolean() })), +}; +`, + ); +} + +function buildRuntimeConfig(projectDir: string): RuntimeConfig { + return { + env: "development", + account: "race.near", + networkId: "testnet", + host: { + name: "host", + url: "http://localhost:3000", + entry: "./src/program.ts", + source: "local", + }, + ui: { + name: "ui", + url: "http://localhost:3003", + entry: "./src/entry.tsx", + source: "local", + localPath: join(projectDir, "ui"), + }, + api: { + name: "api", + url: "http://localhost:3001", + entry: "./src/index.ts", + source: "local", + localPath: join(projectDir, "api"), + }, + plugins: Object.fromEntries( + PLUGIN_KEYS.map((key) => [ + key, + { + name: `race-${key}`, + url: `http://localhost:4110/${key}`, + entry: "./src/index.ts", + source: "local", + localPath: join(projectDir, "plugins", key), + }, + ]), + ), + } as unknown as RuntimeConfig; +} + +describe("codegen artifact generation under concurrent boots", () => { + let projectDir: string; + + beforeAll(() => { + projectDir = mkdtempSync(join(tmpdir(), "bos-codegen-race-")); + writeFileSync(join(projectDir, "bos.config.json"), JSON.stringify(CONFIG)); + const apiDir = join(projectDir, "api"); + writeContractSource(apiDir, "apiBase"); // api/src/contract.ts + for (const key of PLUGIN_KEYS) { + const pluginDir = join(projectDir, "plugins", key); + cpSync(apiDir, pluginDir, { recursive: true }); + } + }); + + afterAll(() => { + rmSync(projectDir, { recursive: true, force: true }); + }); + + it("parallel bridge syncs and resolved-config writes never crash or tear", async () => { + const runtimeConfig = buildRuntimeConfig(projectDir); + + const cycles = 8; + for (let cycle = 0; cycle < cycles; cycle++) { + await Promise.all([ + syncApiContractBridge({ + configDir: projectDir, + runtimeConfig, + apiBaseUrl: "http://localhost:3001", + }), + syncApiContractBridge({ + configDir: projectDir, + runtimeConfig, + apiBaseUrl: "http://localhost:3001", + }), + writeResolvedConfig(projectDir, CONFIG, "development", ["bos://parent.near/app"]), + ]); + + const resolved = JSON.parse( + readFileSync(join(projectDir, ".bos", "bos.resolved-config.json"), "utf-8"), + ) as { _resolved?: { env: string; extendsChain: string[] } }; + expect(resolved._resolved).toMatchObject({ env: "development" }); + } + + for (const key of PLUGIN_KEYS) { + const genPath = join(projectDir, "plugins", key, "src", "lib", "plugins-client.gen.ts"); + expect(existsSync(genPath), `${genPath} must survive concurrent generations`).toBe(true); + expect(readFileSync(genPath, "utf-8")).toContain("export type PluginsClient"); + } + + expect( + readFileSync(join(projectDir, "ui", "src", "lib", "api-types.gen.ts"), "utf-8"), + ).toContain("export type ApiContract"); + }); + + it("sweeps stale gen files of unconfigured plugins without crashing", async () => { + const ghost = join(projectDir, "plugins", "ghost", "src", "lib"); + mkdirSync(ghost, { recursive: true }); + writeFileSync(join(ghost, "plugins-client.gen.ts"), "export type Stale = never;\n"); + + await syncApiContractBridge({ + configDir: projectDir, + runtimeConfig: buildRuntimeConfig(projectDir), + apiBaseUrl: "http://localhost:3001", + }); + + expect(existsSync(join(ghost, "plugins-client.gen.ts"))).toBe(false); + }); +}); diff --git a/packages/everything-dev/tests/unit/db-binding.test.ts b/packages/everything-dev/tests/unit/db-binding.test.ts index a0dc29ae5..fecd639c1 100644 --- a/packages/everything-dev/tests/unit/db-binding.test.ts +++ b/packages/everything-dev/tests/unit/db-binding.test.ts @@ -2,11 +2,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { bindingEnv, resolveDatabaseUrl, workspaceIdentityFromWorkspaceDir } from "../../src/db"; - -const DRIZZLE_KIT_STUDIO_ARGV = ["node", "/repo/node_modules/drizzle-kit/bin.cjs", "studio"]; -const DRIZZLE_KIT_GENERATE_ARGV = ["node", "/repo/node_modules/drizzle-kit/bin.cjs", "generate"]; -const NON_DRIZZLE_KIT_ARGV = ["node", "/repo/scripts/build.ts"]; +import { + bindingEnv, + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromWorkspaceDir, +} from "../../src/db"; function makeWorkspace(): { workspaceDir: string; cleanup: () => void } { const root = mkdtempSync(join(tmpdir(), "everything-dev-binding-")); @@ -26,7 +27,6 @@ describe("resolveDatabaseUrl", () => { const identity = workspaceIdentity(workspaceDir); const url = resolveDatabaseUrl(identity, { env: { WIDGETS_DATABASE_URL: "postgres://direct:value@localhost:5432/x" }, - argv: DRIZZLE_KIT_STUDIO_ARGV, }); expect(url).toBe("postgres://direct:value@localhost:5432/x"); } finally { @@ -48,53 +48,37 @@ describe("resolveDatabaseUrl", () => { ); try { const identity = workspaceIdentity(workspaceDir); - const url = resolveDatabaseUrl(identity, { - env: {}, - argv: DRIZZLE_KIT_STUDIO_ARGV, - }); + const url = resolveDatabaseUrl(identity, { env: {} }); expect(url).toBe("postgres://envfile:value@localhost:5432/widgets_db"); } finally { rmSync(root, { recursive: true, force: true }); } }); - it("throws a loud error naming the secret when a DB-requiring command has no URL", () => { + it("defaults to the persistent pglite data dir when no URL is configured", () => { const { workspaceDir, cleanup } = makeWorkspace(); try { const identity = workspaceIdentity(workspaceDir); - expect(() => - resolveDatabaseUrl(identity, { env: {}, argv: DRIZZLE_KIT_STUDIO_ARGV }), - ).toThrow(/WIDGETS_DATABASE_URL/); - expect(() => - resolveDatabaseUrl(identity, { env: {}, argv: DRIZZLE_KIT_STUDIO_ARGV }), - ).toThrow(/drizzle-kit studio/); + expect(resolveDatabaseUrl(identity, { env: {} })).toBe("pglite:.bos/db"); } finally { cleanup(); } }); +}); - it("returns a pglite placeholder for generate (never connects)", () => { - const { workspaceDir, cleanup } = makeWorkspace(); - try { - const identity = workspaceIdentity(workspaceDir); - expect(resolveDatabaseUrl(identity, { env: {}, argv: DRIZZLE_KIT_GENERATE_ARGV })).toBe( - "pglite:.bos/widgets/:memory:", - ); - } finally { - cleanup(); - } +describe("drizzleCredentialsFor", () => { + it("selects the pglite dialect and data-dir driver tag for pglite URLs", () => { + expect(drizzleCredentialsFor("pglite:.bos/db")).toEqual({ + dialect: "pglite", + dbCredentials: { driver: "pglite", url: ".bos/db" }, + }); }); - it("returns a pglite placeholder outside drizzle-kit invocations", () => { - const { workspaceDir, cleanup } = makeWorkspace(); - try { - const identity = workspaceIdentity(workspaceDir); - expect(resolveDatabaseUrl(identity, { env: {}, argv: NON_DRIZZLE_KIT_ARGV })).toBe( - "pglite:.bos/widgets/:memory:", - ); - } finally { - cleanup(); - } + it("selects the postgresql dialect with a plain DSN otherwise", () => { + expect(drizzleCredentialsFor("postgres://u:p@localhost:5432/db")).toEqual({ + dialect: "postgresql", + dbCredentials: { url: "postgres://u:p@localhost:5432/db" }, + }); }); }); diff --git a/packages/everything-dev/tests/unit/db-driver-pglite-persistence.test.ts b/packages/everything-dev/tests/unit/db-driver-pglite-persistence.test.ts new file mode 100644 index 000000000..224eedb7a --- /dev/null +++ b/packages/everything-dev/tests/unit/db-driver-pglite-persistence.test.ts @@ -0,0 +1,65 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { sql } from "drizzle-orm"; +import { afterEach, describe, expect, it } from "vitest"; +import { createDatabaseDriver } from "../../src/db/driver"; + +const cleanups: Array<() => void> = []; + +afterEach(() => { + for (const cleanup of cleanups.splice(0)) cleanup(); +}); + +function tempDir(name: string): string { + const dir = mkdtempSync(join(tmpdir(), name)); + cleanups.push(() => rmSync(dir, { recursive: true, force: true })); + return dir; +} + +describe("file-backed pglite driver persistence", () => { + it("persists data across driver close/reopen on the same data dir", async () => { + const dir = tempDir("everything-dev-pglite-persist-"); + const url = `pglite:file://${dir}`; + + const first = await createDatabaseDriver(url, {}, undefined); + await first.db.execute(sql`CREATE TABLE probe AS SELECT 'marker-1' AS v`); + await first.close(); + + const second = await createDatabaseDriver(url, {}, undefined); + const read = await second.db.execute<{ v: string }>(sql`SELECT v FROM probe`); + expect(read.rows).toEqual([{ v: "marker-1" }]); + await second.close(); + }); + + it("creates missing directory prefixes recursively for a relative url", async () => { + try { + const write = await createDatabaseDriver("pglite:.bos/db", {}, undefined); + await write.db.execute(sql`CREATE TABLE rel_probe AS SELECT 'marker-2' AS v`); + await write.close(); + + const reopened = await createDatabaseDriver("pglite:.bos/db", {}, undefined); + const read = await reopened.db.execute<{ v: string }>(sql`SELECT v FROM rel_probe`); + expect(read.rows).toEqual([{ v: "marker-2" }]); + await reopened.close(); + } finally { + rmSync(join(".bos", "db"), { recursive: true, force: true }); + } + }); + + it("keeps namespace schema isolation on the shared data dir", async () => { + const dir = tempDir("everything-dev-pglite-ns-"); + const url = `pglite:file://${dir}`; + + const alpha = await createDatabaseDriver(url, {}, "plugin_alpha"); + await alpha.db.execute(sql`CREATE TABLE plugin_alpha.marked AS SELECT 1 AS v`); + await alpha.close(); + + const beta = await createDatabaseDriver(url, {}, "plugin_beta"); + const listed = await beta.db.execute<{ table_name: string }>( + sql`SELECT table_name FROM information_schema.tables WHERE table_schema = 'plugin_beta'`, + ); + expect(listed.rows).toEqual([]); + await beta.close(); + }); +}); diff --git a/packages/everything-dev/tests/unit/db-services.test.ts b/packages/everything-dev/tests/unit/db-services.test.ts index 3c1b5777a..06e1f7079 100644 --- a/packages/everything-dev/tests/unit/db-services.test.ts +++ b/packages/everything-dev/tests/unit/db-services.test.ts @@ -154,11 +154,35 @@ describe("DatabaseBindings.forPluginKey", () => { expect(String(error)).toContain('Plugin "nope" not found'); }); - it("fails loudly naming the missing secret", async () => { + it("falls back to the persistent pglite default naming the secret for a local workspace", async () => { const widgetsDir = tempWorkspace("@everything-dev/widgets-plugin"); const bindings = makeBindings(fixtureRuntimeConfig(widgetsDir), {}); + const binding = await Effect.runPromise(bindings.forPluginKey("widgets")); + expect(binding.url).toBe("pglite:.bos/db"); + expect(binding.source).toBe("local"); + }); + + it("still fails loudly when the declared secret disagrees with the workspace package name", async () => { + const widgetsDir = tempWorkspace("@everything-dev/widgets-plugin"); + const config = fixtureRuntimeConfig(widgetsDir); + config.plugins = { + widgets: { + key: "widgets", + source: "local", + localPath: widgetsDir, + secrets: ["SOMEONE_ELSES_DATABASE_URL"], + }, + }; + const bindings = makeBindings(config, {}); const error = await expectFailure(bindings.forPluginKey("widgets")); expect(String(error)).toContain("WIDGETS_DATABASE_URL"); + expect(String(error)).toContain("disagree"); + }); + + it("still fails for a remote-only plugin with no URL", async () => { + const bindings = makeBindings(fixtureRuntimeConfig("/unused"), {}); + const error = await expectFailure(bindings.forPluginKey("proposals")); + expect(String(error)).toContain("remote plugin"); }); }); diff --git a/packages/everything-dev/tests/unit/docker-autostart.test.ts b/packages/everything-dev/tests/unit/docker-autostart.test.ts deleted file mode 100644 index eec6e09b3..000000000 --- a/packages/everything-dev/tests/unit/docker-autostart.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; -import { - isDockerTestMode, - runDockerComposeUp, - shouldAutoStartDocker, - takeTail, -} from "../../src/infra/docker"; -import type { PreflightFailure } from "../../src/infra/preflight"; - -const command = vi.hoisted(() => vi.fn()); -vi.mock("execa", () => ({ execa: command })); - -describe("runDockerComposeUp", () => { - it("preserves line boundaries and failure diagnostics from both output streams", async () => { - command.mockResolvedValue({ - exitCode: 1, - stdout: "Starting database\n", - stderr: "\u001b[31mDatabase failed\u001b[0m\nCheck credentials\n", - }); - expect(await runDockerComposeUp("/test/project")).toEqual({ - ok: false, - tail: "Starting database\nDatabase failed\nCheck credentials", - }); - }); -}); - -const unreachable = (secret: string): PreflightFailure => ({ - secret, - host: "localhost", - port: 59998, - error: `${secret} points to localhost:59998 but nothing is listening.`, - tcpReachable: false, -}); - -const authFailure = (secret: string): PreflightFailure => ({ - secret, - host: "localhost", - port: 5432, - error: `${secret} at localhost:5432 is reachable but Postgres connection failed.`, - tcpReachable: true, -}); - -const guards = { composeFileExists: true, dockerAvailable: true, testMode: false }; - -describe("shouldAutoStartDocker", () => { - it("starts docker when every failure is an unreachable local service", () => { - expect(shouldAutoStartDocker([unreachable("API_DATABASE_URL")], guards)).toBe(true); - expect( - shouldAutoStartDocker( - [unreachable("API_DATABASE_URL"), unreachable("CACHE_REDIS_URL")], - guards, - ), - ).toBe(true); - }); - - it("never starts docker when any failure is reachable-but-broken (credentials)", () => { - expect(shouldAutoStartDocker([authFailure("API_DATABASE_URL")], guards)).toBe(false); - expect( - shouldAutoStartDocker( - [unreachable("API_DATABASE_URL"), authFailure("AUTH_DATABASE_URL")], - guards, - ), - ).toBe(false); - }); - - it("requires the compose file", () => { - expect( - shouldAutoStartDocker([unreachable("API_DATABASE_URL")], { - ...guards, - composeFileExists: false, - }), - ).toBe(false); - }); - - it("requires docker to be available", () => { - expect( - shouldAutoStartDocker([unreachable("API_DATABASE_URL")], { - ...guards, - dockerAvailable: false, - }), - ).toBe(false); - }); - - it("never starts docker in test mode", () => { - expect( - shouldAutoStartDocker([unreachable("API_DATABASE_URL")], { ...guards, testMode: true }), - ).toBe(false); - }); - - it("does nothing when preflight passed", () => { - expect(shouldAutoStartDocker([], guards)).toBe(false); - }); -}); - -describe("isDockerTestMode", () => { - it("detects the regression-stack env markers", () => { - expect(isDockerTestMode({ BOS_TEST: "1" })).toBe(true); - expect(isDockerTestMode({ NODE_ENV: "test" })).toBe(true); - expect(isDockerTestMode({ BOS_NO_PERSIST_PORTS: "1" })).toBe(true); - expect(isDockerTestMode({ BOS_TEST: "0", NODE_ENV: "development" })).toBe(false); - expect(isDockerTestMode({})).toBe(false); - }); -}); - -describe("takeTail", () => { - it("keeps the last N lines, joined and trimmed", () => { - expect(takeTail(["a", "b", "c"], 2)).toBe("b\nc"); - expect(takeTail(["a", "b", "c"])).toBe("a\nb\nc"); - }); - - it("returns an empty string for no output", () => { - expect(takeTail([], 5)).toBe(""); - }); -}); diff --git a/packages/everything-dev/tests/unit/infra.test.ts b/packages/everything-dev/tests/unit/infra.test.ts index 8afe7ddca..6dd8ad73b 100644 --- a/packages/everything-dev/tests/unit/infra.test.ts +++ b/packages/everything-dev/tests/unit/infra.test.ts @@ -86,10 +86,10 @@ describe("generated env templates", () => { expect(envExample).toContain("# app.api"); expect(envExample).toContain("# app.auth"); // Substrate URLs are opt-in: the example renders them empty with a hint, - // so a fresh `bos dev` with no .env runs on ephemeral pglite. + // so a fresh `bos dev` with no .env picks the persistent .bos/db default. expect(envExample).not.toContain("postgres://everythingdev"); expect(envExample).toContain( - "# Leave empty to run dev on ephemeral pglite; set a postgres:// URL to opt into compose Postgres.", + "# Leave empty for the persistent local PGlite default (.bos/db); set a postgres:// URL for production or shared access.", ); expect(envExample).toContain("BETTER_AUTH_SECRET="); expect(envExample).toContain("# plugins.example"); @@ -100,7 +100,7 @@ describe("generated env templates", () => { expect(existsSync(join(dir, "docker-compose.yml"))).toBe(false); }); - it("writes a committed .env.test with isolated test database URLs", async () => { + it("writes a committed .env.test with in-memory pglite test database URLs", async () => { const dir = mkdtempSync(join(tmpdir(), "bos-env-test-")); tempDirs.push(dir); @@ -108,18 +108,12 @@ describe("generated env templates", () => { const envTest = readFileSync(join(dir, ".env.test"), "utf-8"); expect(envTest).toContain("# app.api"); - expect(envTest).toContain( - "API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db", - ); + expect(envTest).toContain("API_DATABASE_URL=pglite::memory:"); expect(envTest).toContain("# app.auth"); - expect(envTest).toContain( - "AUTH_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5435/auth_test_db", - ); + expect(envTest).toContain("AUTH_DATABASE_URL=pglite::memory:"); expect(envTest).toContain("BETTER_AUTH_SECRET=regression-test-secret-do-not-use-in-production"); expect(envTest).toContain("# plugins.example"); - expect(envTest).toContain( - "EXAMPLE_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db", - ); + expect(envTest).toContain("EXAMPLE_DATABASE_URL=pglite::memory:"); expect(envTest).not.toContain("CORS_ORIGIN="); expect(envTest).not.toContain("PAYMENT_API_URL="); }); diff --git a/packages/everything-dev/tests/unit/infra/planner.test.ts b/packages/everything-dev/tests/unit/infra/planner.test.ts index 164601f94..6cbb6ddf6 100644 --- a/packages/everything-dev/tests/unit/infra/planner.test.ts +++ b/packages/everything-dev/tests/unit/infra/planner.test.ts @@ -223,30 +223,11 @@ describe("buildLaunchSpec", () => { }); describe("buildEnvGenerated", () => { - it("populates CORS_ORIGIN and DB URLs", () => { - const env = buildEnvGenerated( - stubResolvedPorts({ host: 8080 }), - [ - { - secret: "API_DATABASE_URL", - slug: "api", - port: 5432, - dbName: "api", - url: "postgres://u:p@localhost:5432/api", - }, - ], - [ - { - secret: "REDIS_URL", - slug: "cache", - port: 6379, - url: "redis://localhost:6379/0", - }, - ], - ); + it("populates only the generated-origin origins (CORS_ORIGIN/BASE_URL)", () => { + const env = buildEnvGenerated(stubResolvedPorts({ host: 8080 })); expect(env.CORS_ORIGIN).toBe("http://localhost:8080"); - expect(env.API_DATABASE_URL).toBe("postgres://u:p@localhost:5432/api"); - expect(env.REDIS_URL).toBe("redis://localhost:6379/0"); + expect(env.BASE_URL).toBe("http://localhost:8080"); + expect(Object.keys(env)).toEqual(["CORS_ORIGIN", "BASE_URL"]); }); }); @@ -460,7 +441,7 @@ describe("planInfra zero-config substrate (plan 055 ticket 11)", () => { expect(plan.envGenerated.CORS_ORIGIN).toBeDefined(); }); - it("partial operator supply generates only the missing conventional URL", async () => { + it("partial operator supply generates nothing conventional — substrate URLs are operator-owned", async () => { const plan = await Effect.runPromise( planInfra({ configDir: tempDir, @@ -472,7 +453,10 @@ describe("planInfra zero-config substrate (plan 055 ticket 11)", () => { }).pipe(Effect.provide(Layer.mergeAll(PortAllocatorLive))), ); - expect(plan.envGenerated.API_DATABASE_URL).toBeUndefined(); - expect(plan.envGenerated.AUTH_DATABASE_URL).toContain("5433/auth_db"); + expect(Object.keys(plan.envGenerated)).toEqual( + expect.arrayContaining(["CORS_ORIGIN", "BASE_URL"]), + ); + expect(Object.keys(plan.envGenerated)).not.toContain("API_DATABASE_URL"); + expect(Object.keys(plan.envGenerated)).not.toContain("AUTH_DATABASE_URL"); }); }); diff --git a/packages/everything-dev/tests/unit/legacy-databases.test.ts b/packages/everything-dev/tests/unit/legacy-databases.test.ts new file mode 100644 index 000000000..4aa428d36 --- /dev/null +++ b/packages/everything-dev/tests/unit/legacy-databases.test.ts @@ -0,0 +1,155 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Effect } from "effect"; +import { describe, expect, it } from "vitest"; +import { + migrateLegacyComposeState, + rewrittenLegacyDatabaseValues, +} from "../../src/env/legacy-databases"; + +function makeProject(): { dir: string; cleanup: () => void } { + const dir = mkdtempSync(join(tmpdir(), "everything-dev-legacy-")); + return { dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) }; +} + +function runMigration(dir: string) { + return Effect.runSync(migrateLegacyComposeState(dir)); +} + +const composeScaffold = `services: + postgres-api: + image: ghcr.io/nearbuilders/mirror-postgres:17-alpine + ports: + - "5432:5432" + postgres-auth: + image: ghcr.io/nearbuilders/mirror-postgres:17-alpine + ports: + - "5433:5432" +`; + +const composeCustom = `services: + mydb: + image: postgres:17-alpine + ports: + - "6999:5432" +`; + +describe("rewrittenLegacyDatabaseValues", () => { + it("rewrites the conventional dev URLs to the persistent pglite default", () => { + const rewritten = rewrittenLegacyDatabaseValues({ + file: { + API_DATABASE_URL: "postgres://everythingdev:everythingdev@localhost:5432/api_db", + AUTH_DATABASE_URL: "postgres://everythingdev:everythingdev@localhost:5433/auth_db", + }, + raw: + "API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5432/api_db\n" + + "AUTH_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5433/auth_db\n", + }); + expect(rewritten.get("API_DATABASE_URL")).toBe("pglite:.bos/db"); + expect(rewritten.get("AUTH_DATABASE_URL")).toBe("pglite:.bos/db"); + }); + + it("drifted convention ports still rewrite by database name", () => { + const rewritten = rewrittenLegacyDatabaseValues({ + file: { API_DATABASE_URL: "postgres://everythingdev:everythingdev@localhost:5532/api_db" }, + raw: "API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5532/api_db\n", + }); + expect(rewritten.get("API_DATABASE_URL")).toBe("pglite:.bos/db"); + }); + + it("rewrites the conventional test URLs to in-memory pglite", () => { + const rewritten = rewrittenLegacyDatabaseValues({ + file: { + API_DATABASE_URL: "postgres://everythingdev:everythingdev@localhost:5434/api_test_db", + AUTH_DATABASE_URL: "postgres://everythingdev:everythingdev@localhost:5435/auth_test_db", + }, + raw: + "API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5434/api_test_db\n" + + "AUTH_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5435/auth_test_db\n", + }); + expect(rewritten.get("API_DATABASE_URL")).toBe("pglite::memory:"); + expect(rewritten.get("AUTH_DATABASE_URL")).toBe("pglite::memory:"); + }); + + it("leaves operator-owned postgres URLs alone", () => { + const custom = "postgres://myuser:mypass@localhost:5432/api_db"; + const remote = "postgres://u:p@ep-neon.example.com/api_db"; + const train = "postgres://rider:pass@127.0.0.1:5432/different_db"; + const rewritten = rewrittenLegacyDatabaseValues({ + file: { API_DATABASE_URL: custom, OTHER_DATABASE_URL: remote, TRAIN: train }, + raw: `API_DATABASE_URL=${custom}\nOTHER_DATABASE_URL=${remote}\nTRAIN=${train}\n`, + }); + expect(rewritten.size).toBe(0); + }); + + it("ignores non-postgres schemes and pglite URLs", () => { + const rewritten = rewrittenLegacyDatabaseValues({ + file: { API_DATABASE_URL: "pglite:.bos/db" }, + raw: "API_DATABASE_URL=pglite:.bos/db\n", + }); + expect(rewritten.size).toBe(0); + }); +}); + +describe("migrateLegacyComposeState", () => { + it("rewrites .env lines and relocates the scaffold compose file", () => { + const { dir, cleanup } = makeProject(); + try { + writeFileSync( + join(dir, ".env"), + "API_DATABASE_URL=postgres://everythingdev:everythingdev@localhost:5432/api_db\n" + + "CUSTOM_DATABASE_URL=postgres://myuser:mypass@localhost:5432/custom\n", + ); + writeFileSync(join(dir, "docker-compose.yml"), composeScaffold); + + const result = runMigration(dir); + + expect(result.composeRelocated).toBe(true); + expect(result.databaseUrlDrift).toEqual([ + { + key: "API_DATABASE_URL", + from: "postgres://everythingdev:everythingdev@localhost:5432/api_db", + to: "pglite:.bos/db", + }, + ]); + + const env = readFileSync(join(dir, ".env"), "utf-8"); + expect(env).toContain("API_DATABASE_URL=pglite:.bos/db"); + expect(env).toContain("CUSTOM_DATABASE_URL=postgres://myuser:mypass@localhost:5432/custom"); + expect( + readFileSync(join(dir, ".bos", "compose-retired", "docker-compose.yml"), "utf-8"), + ).toBe(composeScaffold); + } finally { + cleanup(); + } + }); + + it("is idempotent: a second run is a no-op", () => { + const { dir, cleanup } = makeProject(); + try { + writeFileSync(join(dir, ".env"), "API_DATABASE_URL=pglite:.bos/db\n"); + writeFileSync(join(dir, "docker-compose.yml"), composeScaffold); + + const first = runMigration(dir); + expect(first.composeRelocated).toBe(true); + const second = runMigration(dir); + expect(second.composeRelocated).toBe(false); + expect(second.databaseUrlDrift).toEqual([]); + } finally { + cleanup(); + } + }); + + it("does not relocate an operator-authored compose file", () => { + const { dir, cleanup } = makeProject(); + try { + writeFileSync(join(dir, "docker-compose.yml"), composeCustom); + const result = runMigration(dir); + expect(result.composeRelocated).toBe(false); + expect(readFileSync(join(dir, "docker-compose.yml"), "utf-8")).toBe(composeCustom); + } finally { + cleanup(); + } + }); +}); diff --git a/packages/everything-dev/tests/unit/origin-truth.test.ts b/packages/everything-dev/tests/unit/origin-truth.test.ts index eb045d50e..4b7152bb5 100644 --- a/packages/everything-dev/tests/unit/origin-truth.test.ts +++ b/packages/everything-dev/tests/unit/origin-truth.test.ts @@ -32,11 +32,11 @@ describe("preflight credentialed URL parsing", () => { expect(failures).toEqual([]); }); - it("gives API/AUTH failures the docker-compose hint", async () => { + it("gives API/AUTH failures the persistent-pglite fallback hint", async () => { const env = { API_DATABASE_URL: "postgres://u:p@localhost:59998/api_db" }; const failures = await Effect.runPromise(preflightLocalInfra(env)); expect(failures.length).toBe(1); - expect(failures[0]?.error).toContain("docker compose up -d --wait"); + expect(failures[0]?.error).toContain("persistent .bos/db PGlite"); expect(failures[0]?.tcpReachable).toBe(false); }, 20000); }); diff --git a/plugins/_template/drizzle.config.ts b/plugins/_template/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/plugins/_template/drizzle.config.ts +++ b/plugins/_template/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/plugins/_template/plugin.dev.ts b/plugins/_template/plugin.dev.ts index 48b6032f3..11169194c 100644 --- a/plugins/_template/plugin.dev.ts +++ b/plugins/_template/plugin.dev.ts @@ -11,7 +11,7 @@ export default { }, secrets: { TEMPLATE_API_KEY: process.env.TEMPLATE_API_KEY || "dev-key-12345", - TEMPLATE_DATABASE_URL: process.env.TEMPLATE_DATABASE_URL || "pglite:.bos/_template/:memory:", + TEMPLATE_DATABASE_URL: process.env.TEMPLATE_DATABASE_URL || "pglite:.bos/db", }, } satisfies PluginConfigInput, }; diff --git a/plugins/_template/src/index.ts b/plugins/_template/src/index.ts index dfd5f75a0..e69116611 100644 --- a/plugins/_template/src/index.ts +++ b/plugins/_template/src/index.ts @@ -68,8 +68,10 @@ export default createPlugin.withPlugins()({ TEMPLATE_API_KEY: z.string().min(1, "TEMPLATE_API_KEY is required").default("template-dev-key"), TEMPLATE_DATABASE_URL: z .string() - .default("pglite:.bos/_template/:memory:") - .describe("Database connection string. Use pglite: for local, postgres:// for production."), + .default("pglite:.bos/db") + .describe( + "Database connection string. pglite: is a persistent local data dir (.bos/db); postgres:// for production or shared access.", + ), }), context: ContextSchema, diff --git a/plugins/auth/README.md b/plugins/auth/README.md index c0c8d5361..48bbe126a 100644 --- a/plugins/auth/README.md +++ b/plugins/auth/README.md @@ -117,8 +117,8 @@ The plugin itself does **not** read `process.env`. The host passes all configura The plugin uses PostgreSQL via `pg` in production and `@electric-sql/pglite` for local development and tests. `pg` is externalized in the Module Federation bundle and must be provided by the host at runtime. ```bash -# Local development (zero-config, no Docker needed) -AUTH_DATABASE_URL=pglite:./auth-local.db +# Local development (zero-config, no Docker needed — persistent PGlite data dir) +AUTH_DATABASE_URL=pglite:.bos/db # Production (Railway, etc.) AUTH_DATABASE_URL=postgres://user:pass@host:5432/dbname @@ -218,7 +218,7 @@ export default { }, }, secrets: { - AUTH_DATABASE_URL: "pglite:./.local/auth.db", + AUTH_DATABASE_URL: "pglite:.bos/db", BETTER_AUTH_SECRET: "dev-only-secret", }, } satisfies PluginConfigInput, diff --git a/plugins/auth/drizzle.config.ts b/plugins/auth/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/plugins/auth/drizzle.config.ts +++ b/plugins/auth/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/plugins/auth/plugin.dev.ts b/plugins/auth/plugin.dev.ts index 03ec28123..c95838446 100644 --- a/plugins/auth/plugin.dev.ts +++ b/plugins/auth/plugin.dev.ts @@ -149,7 +149,7 @@ export default { }, }, secrets: { - AUTH_DATABASE_URL: process.env.AUTH_DATABASE_URL || "pglite:.bos/auth/:memory:", + AUTH_DATABASE_URL: process.env.AUTH_DATABASE_URL || "pglite:.bos/db", BETTER_AUTH_SECRET: process.env.BETTER_AUTH_SECRET || "dev-only-secret-do-not-use-in-production", GITHUB_CLIENT_SECRET: process.env.GITHUB_CLIENT_SECRET, @@ -161,6 +161,7 @@ export default { RESEND_API_KEY: process.env.RESEND_API_KEY, NEAR_RELAYER_PRIVATE_KEY_MAINNET: process.env.NEAR_RELAYER_PRIVATE_KEY_MAINNET, NEAR_RELAYER_PRIVATE_KEY_TESTNET: process.env.NEAR_RELAYER_PRIVATE_KEY_TESTNET, + AUTH_BOOTSTRAP_ADMIN_API_KEY: process.env.AUTH_BOOTSTRAP_ADMIN_API_KEY, }, } satisfies PluginConfigInput, }; diff --git a/plugins/auth/src/bootstrap-admin.ts b/plugins/auth/src/bootstrap-admin.ts new file mode 100644 index 000000000..1927df571 --- /dev/null +++ b/plugins/auth/src/bootstrap-admin.ts @@ -0,0 +1,116 @@ +import { createHash, randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { Data, Effect } from "effect"; +import type { Database } from "./db/layer"; +import * as schema from "./db/schema"; + +export class BootstrapAdminFailure extends Data.TaggedError("BootstrapAdminFailure")<{ + readonly message: string; +}> {} + +const BOOTSTRAP_ADMIN_EMAIL = "bootstrap-admin@regression.internal"; +const BOOTSTRAP_NAME = "bootstrap-admin"; +const BOOTSTRAP_KEY_PREFIX = "api_"; +const BOOTSTRAP_KEY_START_LENGTH = 8; + +/** + * The base64url translation of @better-auth/api-key's default key hasher — + * `createHash("SHA-256").digest(key)` → unpadded base64url — so a row written + * here validates through the plugin's normal `verifyApiKey` path. + */ +function hashBootstrapKey(secret: string): string { + return createHash("sha256").update(secret).digest().toString("base64url"); +} + +const dbQuery = (run: () => Promise) => + Effect.tryPromise({ + try: run, + catch: (cause) => + new BootstrapAdminFailure({ + message: `bootstrap admin step failed: ${cause instanceof Error ? cause.message : String(cause)}`, + }), + }); + +/** + * Dev/test-only seam (ADR 0038 addendum): when `AUTH_BOOTSTRAP_ADMIN_API_KEY` + * is set, boot-time seeding via the API needs a platform admin + real API key + * row before any HTTP helper can run. The inserted key verifies through the + * stock `x-api-key` → `verifyApiKey` path; nothing else about auth changes. + * + * The environment's presence of the secret is the explicit opt-in — no + * production deployment should ever set this variable (the boot log says so). + * Any failure here fails plugin init loudly: a regression run with a half + * bootstrapped admin seam would only fail later, confusingly. + */ +export function ensureBootstrapAdmin({ db, secret }: { db: Database; secret: string }) { + return Effect.gen(function* () { + const now = new Date(); + let user = yield* dbQuery(() => + db.query.user.findFirst({ where: eq(schema.user.email, BOOTSTRAP_ADMIN_EMAIL) }), + ); + + if (!user) { + yield* dbQuery(() => + db + .insert(schema.user) + .values({ + id: randomUUID(), + name: BOOTSTRAP_NAME, + email: BOOTSTRAP_ADMIN_EMAIL, + emailVerified: true, + role: "admin", + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing(), + ); + user = yield* dbQuery(() => + db.query.user.findFirst({ where: eq(schema.user.email, BOOTSTRAP_ADMIN_EMAIL) }), + ); + } + + if (!user) { + return yield* new BootstrapAdminFailure({ + message: "bootstrap admin user could not be created", + }); + } + + const hashed = hashBootstrapKey(secret); + const existing = yield* dbQuery(() => + db.query.apikey.findFirst({ where: eq(schema.apikey.key, hashed) }), + ); + + if (existing) { + if ( + existing.referenceId === user.id && + existing.configId === "user-keys" && + existing.enabled !== false + ) { + return; + } + return yield* new BootstrapAdminFailure({ + message: + "AUTH_BOOTSTRAP_ADMIN_API_KEY collides with an existing API key — choose a distinct value", + }); + } + + yield* dbQuery(() => + db + .insert(schema.apikey) + .values({ + id: randomUUID(), + configId: "user-keys", + name: "bootstrap admin", + start: secret.slice(0, Math.min(BOOTSTRAP_KEY_START_LENGTH, secret.length)), + referenceId: user.id, + prefix: BOOTSTRAP_KEY_PREFIX, + key: hashed, + enabled: true, + rateLimitEnabled: false, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing(), + ); + }); +} diff --git a/plugins/auth/src/config-schemas.ts b/plugins/auth/src/config-schemas.ts index af954c299..e74d458e9 100644 --- a/plugins/auth/src/config-schemas.ts +++ b/plugins/auth/src/config-schemas.ts @@ -149,8 +149,10 @@ export const authVariablesSchema = z.object({ export const authSecretsSchema = z.object({ AUTH_DATABASE_URL: z .string() - .default("pglite:.bos/auth/:memory:") - .describe("Database connection string. Use pglite: for local, postgres:// for production."), + .default("pglite:.bos/db") + .describe( + "Database connection string. pglite: is a persistent local data dir (.bos/db); postgres:// for production or shared access.", + ), BETTER_AUTH_SECRET: z.string(), GITHUB_CLIENT_SECRET: z.string().optional(), GOOGLE_CLIENT_SECRET: z.string().optional(), @@ -161,6 +163,13 @@ export const authSecretsSchema = z.object({ NEAR_RELAYER_PRIVATE_KEY_MAINNET: z.string().optional(), NEAR_RELAYER_PRIVATE_KEY_TESTNET: z.string().optional(), RESEND_API_KEY: z.string().optional(), + AUTH_BOOTSTRAP_ADMIN_API_KEY: z + .string() + .optional() + .describe( + "Dev/test-only: boot the instance with a platform admin + API key carrying this exact secret, " + + "so regression fixtures can seed through the admin routes. Never set in production.", + ), }); export type AuthPluginVariables = z.infer; diff --git a/plugins/auth/src/index.ts b/plugins/auth/src/index.ts index 927aab2f1..a6cfdf808 100644 --- a/plugins/auth/src/index.ts +++ b/plugins/auth/src/index.ts @@ -5,6 +5,7 @@ import { z } from "zod"; export type { AuthServices } from "./auth-export"; import { createAuthInstance } from "./auth-instance"; +import { ensureBootstrapAdmin } from "./bootstrap-admin"; import { normalizeAuthConfig } from "./config"; import { authSecretsSchema, authVariablesSchema } from "./config-schemas"; import { contract } from "./contract"; @@ -66,6 +67,16 @@ export default createPlugin.withPlugins()({ yield* Effect.log("[Auth] Better Auth instance created"); + if (config.secrets.AUTH_BOOTSTRAP_ADMIN_API_KEY) { + yield* ensureBootstrapAdmin({ + db, + secret: config.secrets.AUTH_BOOTSTRAP_ADMIN_API_KEY, + }); + yield* Effect.logWarning( + "[Auth] bootstrap admin enabled via AUTH_BOOTSTRAP_ADMIN_API_KEY — dev/test seeding only, never set in production", + ); + } + return Layer.mergeAll( buildAuthServicesLayer({ auth, diff --git a/plugins/auth/src/service-wiring.ts b/plugins/auth/src/service-wiring.ts index bbb4bb83d..2e2c32aa2 100644 --- a/plugins/auth/src/service-wiring.ts +++ b/plugins/auth/src/service-wiring.ts @@ -72,6 +72,13 @@ export function buildAuthServicesLayer(services: PluginServices) { ), Layer.succeed(PasskeysService, buildPasskeysService({ auth: services.auth })), Layer.succeed(ApiKeysService, buildApiKeysService({ auth: services.auth, db: services.db })), - Layer.succeed(AdminUsersService, buildAdminUsersService({ auth: services.auth })), + Layer.succeed( + AdminUsersService, + buildAdminUsersService({ + auth: services.auth, + db: services.db, + apiKeyHeaders: services.apiKeyHeaders, + }), + ), ); } diff --git a/plugins/auth/src/services/admin-users.ts b/plugins/auth/src/services/admin-users.ts index b2fb630f1..860fc3deb 100644 --- a/plugins/auth/src/services/admin-users.ts +++ b/plugins/auth/src/services/admin-users.ts @@ -1,9 +1,11 @@ +import { eq } from "drizzle-orm"; import { Context, Effect } from "effect"; -import { attemptAuth } from "../attempts"; +import { attemptAuth, attemptDb } from "../attempts"; import type { InferInput, InferOutput } from "../contract"; +import * as schema from "../db/schema"; import { toAdminSession, toAdminUser } from "../serializers"; import type { PluginServices } from "../service-types"; -import type { AuthFailure } from "./domain-errors"; +import { type AuthFailure, DbError } from "./domain-errors"; type Auth = PluginServices["auth"]; @@ -59,7 +61,15 @@ export class AdminUsersService extends Context.Service< } >()("auth/AdminUsersService") {} -export const buildAdminUsersService = ({ auth }: { auth: Auth }) => +export const buildAdminUsersService = ({ + auth, + db, + apiKeyHeaders, +}: { + auth: Auth; + db: PluginServices["db"]; + apiKeyHeaders: string[]; +}) => AdminUsersService.of({ listUsers: (headers, input) => attemptAuth(() => @@ -84,10 +94,30 @@ export const buildAdminUsersService = ({ auth }: { auth: Auth }) => Effect.map(toAdminUser), ), - setRole: (headers, userId, role) => - attemptAuth(() => auth.api.setRole({ headers, body: { userId, role } })).pipe( - Effect.map((result) => toAdminUser(result.user)), - ), + setRole: (headers, userId, role) => { + // better-auth's admin endpoints read sessions authoritatively from the + // server session store and null out the api-key hook's hydration, so + // they can never authorize an API-key caller (UNAUTHORIZED 401). For + // API-key callers the typed requireAdmin guard has already verified + // the caller's admin role — perform the update directly. + const apiKeyCaller = apiKeyHeaders.some((name) => Boolean(headers.get(name.toLowerCase()))); + if (!apiKeyCaller) { + return attemptAuth(() => auth.api.setRole({ headers, body: { userId, role } })).pipe( + Effect.map((result) => toAdminUser(result.user)), + ); + } + return attemptDb(async () => { + const [updated] = await db + .update(schema.user) + .set({ role }) + .where(eq(schema.user.id, userId)) + .returning(); + if (!updated) { + throw new DbError({ message: `admin set role: user not found: ${userId}` }); + } + return updated; + }).pipe(Effect.map(toAdminUser)); + }, banUser: (headers, input) => attemptAuth(() => diff --git a/plugins/proposals/drizzle.config.ts b/plugins/proposals/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/plugins/proposals/drizzle.config.ts +++ b/plugins/proposals/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/plugins/proposals/plugin.dev.ts b/plugins/proposals/plugin.dev.ts index 5b954b643..88364c56d 100644 --- a/plugins/proposals/plugin.dev.ts +++ b/plugins/proposals/plugin.dev.ts @@ -11,8 +11,7 @@ export default { privatePluginIds: ["nearcatalog"], }, secrets: { - PROPOSALS_DATABASE_URL: - process.env.PROPOSALS_DATABASE_URL || "pglite:.bos/proposals/:memory:", + PROPOSALS_DATABASE_URL: process.env.PROPOSALS_DATABASE_URL || "pglite:.bos/db", }, } satisfies PluginConfigInput, }; diff --git a/plugins/proposals/src/index.ts b/plugins/proposals/src/index.ts index 6f3b10a4c..1d053051a 100644 --- a/plugins/proposals/src/index.ts +++ b/plugins/proposals/src/index.ts @@ -37,7 +37,7 @@ export default createPlugin({ }), secrets: z.object({ - PROPOSALS_DATABASE_URL: z.string().default("pglite:.bos/proposals/:memory:"), + PROPOSALS_DATABASE_URL: z.string().default("pglite:.bos/db"), }), context: ProposalContextSchema, diff --git a/plugins/votes/drizzle.config.ts b/plugins/votes/drizzle.config.ts index 6b2673306..0d4523f53 100644 --- a/plugins/votes/drizzle.config.ts +++ b/plugins/votes/drizzle.config.ts @@ -5,17 +5,20 @@ * issue: https://github.com/NEARBuilders/everything-dev/issues */ import { defineConfig } from "drizzle-kit"; -import { resolveDatabaseUrl, workspaceIdentityFromModuleUrl } from "everything-dev/db"; +import { + drizzleCredentialsFor, + resolveDatabaseUrl, + workspaceIdentityFromModuleUrl, +} from "everything-dev/db"; const identity = workspaceIdentityFromModuleUrl(import.meta.url); +const credentials = drizzleCredentialsFor(resolveDatabaseUrl(identity)); export default defineConfig({ schema: "./src/db/schema.ts", out: "./src/db/migrations", - dialect: "postgresql", - dbCredentials: { - url: resolveDatabaseUrl(identity), - }, + dialect: credentials.dialect, + dbCredentials: credentials.dbCredentials, migrations: { schema: identity.journal.schema, table: identity.journal.table, diff --git a/plugins/votes/plugin.dev.ts b/plugins/votes/plugin.dev.ts index adcf6d5f9..059f93090 100644 --- a/plugins/votes/plugin.dev.ts +++ b/plugins/votes/plugin.dev.ts @@ -9,7 +9,7 @@ export default { config: { variables: {}, secrets: { - VOTES_DATABASE_URL: process.env.VOTES_DATABASE_URL || "pglite:.bos/votes/:memory:", + VOTES_DATABASE_URL: process.env.VOTES_DATABASE_URL || "pglite:.bos/db", }, } satisfies PluginConfigInput, }; diff --git a/plugins/votes/src/index.ts b/plugins/votes/src/index.ts index d22d8a89b..af88dc7e0 100644 --- a/plugins/votes/src/index.ts +++ b/plugins/votes/src/index.ts @@ -11,7 +11,7 @@ export default createPlugin({ variables: z.object({}), secrets: z.object({ - VOTES_DATABASE_URL: z.string().default("pglite:.bos/votes/:memory:"), + VOTES_DATABASE_URL: z.string().default("pglite:.bos/db"), }), context: ContextSchema, diff --git a/tests/regression/browser/helpers/global-setup.ts b/tests/regression/browser/helpers/global-setup.ts index 8343434ea..7361c0272 100644 --- a/tests/regression/browser/helpers/global-setup.ts +++ b/tests/regression/browser/helpers/global-setup.ts @@ -1,16 +1,5 @@ -import fs from "node:fs"; -import path from "node:path"; import type { PluginLoadFailureInfo } from "every-plugin/errors"; -import { createAuthTestInstance } from "../../lib/auth-test-instance.ts"; -import { seedMemberFixtures } from "../../lib/member-seed.ts"; -import { migrateTestDatabase } from "../../lib/migrate-test-db.mjs"; -import { computeRegressionEnv } from "../../lib/regression-env.mjs"; - -const ADMIN_COOKIES_PATH = ".bos/regression/admin-cookies.json"; -const LOGOUT_COOKIES_PATH = ".bos/regression/logout-cookies.json"; -const ADMIN_SEED_PATH = ".bos/regression/admin-seed.json"; -const ADMIN_NAME = "admin.near"; -const LOGOUT_NAME = "logout.near"; +import { seedRegressionFixtures } from "../../lib/http-seed.mjs"; /** * The stack serves /health 200 even when a plugin failed to load (degraded @@ -37,66 +26,11 @@ async function failOnPluginLoadFailures(baseUrl: string) { } export default async function globalSetup() { - const regressionEnv = computeRegressionEnv(); - const authDatabaseUrl = regressionEnv.dbUrls.AUTH_DATABASE_URL ?? ""; - const secret = regressionEnv.authSecret; - + const regressionEnv = (await import("../../lib/regression-env.mjs")).computeRegressionEnv(); await failOnPluginLoadFailures(regressionEnv.baseUrl); - const { test } = await createAuthTestInstance({ authDatabaseUrl, secret }); - - const unique = `${process.pid}`; - const admin = await test.saveUser( - test.createUser({ - email: `regression-admin-${unique}@regression.test`, - name: ADMIN_NAME, - role: "admin", - emailVerified: true, - }), - ); - - const orgAName = `regression-admin-org-a-${unique}`; - const orgBName = `regression-admin-org-b-${unique}`; - const orgA = await test.saveOrganization( - test.createOrganization({ name: orgAName, slug: orgAName }), - ); - const orgB = await test.saveOrganization( - test.createOrganization({ name: orgBName, slug: orgBName }), - ); - await test.addMember({ userId: admin.id, organizationId: orgA.id, role: "admin" }); - await test.addMember({ userId: admin.id, organizationId: orgB.id, role: "admin" }); - - const cookies = await test.getCookies({ userId: admin.id, domain: "localhost" }); - - const logoutUser = await test.saveUser( - test.createUser({ - email: `regression-logout-${unique}@regression.test`, - name: LOGOUT_NAME, - emailVerified: true, - }), - ); - const logoutCookies = await test.getCookies({ userId: logoutUser.id, domain: "localhost" }); - - const cookiesResolved = path.resolve(process.cwd(), ADMIN_COOKIES_PATH); - fs.mkdirSync(path.dirname(cookiesResolved), { recursive: true }); - fs.writeFileSync(cookiesResolved, JSON.stringify(cookies, null, 2)); - console.log(`[global-setup] wrote ${cookiesResolved}`); - - const logoutCookiesResolved = path.resolve(process.cwd(), LOGOUT_COOKIES_PATH); - fs.writeFileSync(logoutCookiesResolved, JSON.stringify(logoutCookies, null, 2)); - console.log(`[global-setup] wrote ${logoutCookiesResolved}`); - - const seedResolved = path.resolve(process.cwd(), ADMIN_SEED_PATH); - fs.writeFileSync( - seedResolved, - JSON.stringify({ adminName: ADMIN_NAME, logoutName: LOGOUT_NAME, orgAName, orgBName }, null, 2), - ); - console.log(`[global-setup] wrote ${seedResolved}`); - - await migrateTestDatabase({ - migrationsDir: "api/src/db/migrations", - databaseUrl: regressionEnv.dbUrls.API_DATABASE_URL, + await seedRegressionFixtures({ + log: (message) => console.log(`[global-setup] ${message}`), }); - await seedMemberFixtures({ authDatabaseUrl, secret }); - console.log(`[global-setup] refreshed member fixtures`); + console.log(`[global-setup] fixtures refreshed (HTTP seeding)`); } diff --git a/tests/regression/browser/specs/onboard-rescan.spec.ts b/tests/regression/browser/specs/onboard-rescan.spec.ts index 3939b5ebd..fcbf53788 100644 --- a/tests/regression/browser/specs/onboard-rescan.spec.ts +++ b/tests/regression/browser/specs/onboard-rescan.spec.ts @@ -1,77 +1,106 @@ -import { expect, test } from "@playwright/test"; -import { and, eq } from "drizzle-orm"; -import * as authSchema from "../../../../plugins/auth/src/db/schema.ts"; -import { createAuthTestInstance } from "../../lib/auth-test-instance.ts"; +import { expect, type Page, test } from "@playwright/test"; +import { + type CookieJar, + createOrganization, + getSessionBetterAuth, + newAnonymousSession, + postBetterAuth, + postRpc, +} from "../../lib/http-seed.mjs"; import { computeRegressionEnv } from "../../lib/regression-env.mjs"; import { collectErrors } from "../helpers/page-ready"; -type Fixtures = Awaited>; - +/** + * The onboarding rescan journey (ADR 0038 addendum): the stack database is + * in-process pglite, so every fixture is minted over the public API surface — + * anonymous sign-ins, org creation + approval through the admin routes, and + * the "member removed but stale active-org" state produced by the organizer + * removing the joiner (`removeMember` leaves the removed user's session + * untouched, which is exactly the state the rescan must recover from). + */ const regressionEnv = computeRegressionEnv(); function cookieHeader(cookies: Array<{ name: string; value: string }>) { return cookies.map((cookie) => `${cookie.name}=${cookie.value}`).join("; "); } -async function createCode(fixtures: Fixtures, unique: string) { - const organizer = await fixtures.test.saveUser( - fixtures.test.createUser({ - email: `onboard-organizer-${unique}@regression.test`, - name: "Onboard Organizer", - emailVerified: true, +async function rpcGetSession(jar: CookieJar) { + const response = await fetch(`${regressionEnv.baseUrl}/api/rpc/auth/getSession`, { + method: "POST", + headers: { "content-type": "application/json", cookie: cookieHeader(jar.playwrightCookies()) }, + body: JSON.stringify({ json: {} }), + }); + if (!response.ok) { + throw new Error(`POST /api/rpc/auth/getSession failed: ${response.status}`); + } + const { json } = (await response.json()) as { + json: { session: { activeOrganizationId: string | null } | null }; + }; + return json; +} + +async function removeMemberAsOrganizer(organizer: CookieJar, joinerEmail: string, orgId: string) { + const response = await fetch(`${regressionEnv.baseUrl}/api/rpc/auth/removeMember`, { + method: "POST", + headers: { + "content-type": "application/json", + cookie: cookieHeader(organizer.playwrightCookies()), + }, + body: JSON.stringify({ + json: { memberIdOrEmail: joinerEmail, organizationId: orgId }, }), - ); - const orgName = `onboard-rescan-${unique}`; - const org = await fixtures.test.saveOrganization( - fixtures.test.createOrganization({ name: orgName, slug: orgName }), - ); - await fixtures.test.addMember({ userId: organizer.id, organizationId: org.id, role: "owner" }); - const cookies = await fixtures.test.getCookies({ userId: organizer.id, domain: "localhost" }); + }); + if (!response.ok) { + throw new Error(`removeMember failed: ${response.status} ${await response.text()}`); + } +} +async function createOnboardingCode(organizer: CookieJar, org: { id: string }, unique: string) { const response = await fetch(`${regressionEnv.baseUrl}/api/rpc/auth/createOnboardingCode`, { method: "POST", - headers: { "content-type": "application/json", cookie: cookieHeader(cookies) }, + headers: { + "content-type": "application/json", + cookie: cookieHeader(organizer.playwrightCookies()), + }, body: JSON.stringify({ - json: { eventId: `rescan-${unique}`, eventName: "Rescan Night", organizationId: org.id }, + json: { + eventId: `rescan-${unique}`, + eventName: "Rescan Night", + organizationId: org.id, + }, }), }); if (!response.ok) { throw new Error(`createOnboardingCode failed: ${response.status} ${await response.text()}`); } const { json } = (await response.json()) as { json: { code: string } }; - return { org, code: json.code }; + return json.code; } -test.describe("onboard rescan", () => { - let fixtures: Fixtures; - - test.beforeAll(async () => { - fixtures = await createAuthTestInstance({ - authDatabaseUrl: regressionEnv.dbUrls.AUTH_DATABASE_URL ?? "", - secret: regressionEnv.authSecret, - }); - }); - - test.afterAll(async () => { - await fixtures.close(); - }); +async function addSessionCookies(page: Page, jar: CookieJar) { + await page.context().addCookies(jar.playwrightCookies()); +} +test.describe("onboard rescan", () => { test("a member who left is told so on rescan, not that they rejoined", async ({ page }) => { const pageErrors = collectErrors(page); const unique = `${process.pid}-${Date.now()}`; - const { org, code } = await createCode(fixtures, unique); - const joiner = await fixtures.test.saveUser( - fixtures.test.createUser({ - email: `onboard-joiner-${unique}@regression.test`, - name: "Rescan Tester", - emailVerified: true, - }), - ); - await page - .context() - .addCookies(await fixtures.test.getCookies({ userId: joiner.id, domain: "localhost" })); + const organizerJar = await newAnonymousSession(regressionEnv, "Onboard Organizer"); + const org = await createOrganization(regressionEnv, organizerJar, `onboard-rescan-${unique}`); + await postRpc(regressionEnv, "auth/reviewOrganization", { + organizationId: org.id, + decision: "approve", + }); + const code = await createOnboardingCode(organizerJar, org, unique); + + const joinerJar = await newAnonymousSession(regressionEnv, "Rescan Tester"); + const joiner = await getSessionBetterAuth(regressionEnv.baseUrl, joinerJar); + if (!joiner?.user?.email) { + throw new Error("joiner session produced no user email"); + } + await addSessionCookies(page, joinerJar); await page.goto(`/onboard?code=${code}`); await expect(page.getByTestId("onboard.continue-on-computer")).toBeVisible({ timeout: 20000, @@ -83,15 +112,17 @@ test.describe("onboard rescan", () => { timeout: 20000, }); - await fixtures.db - .delete(authSchema.member) - .where( - and(eq(authSchema.member.userId, joiner.id), eq(authSchema.member.organizationId, org.id)), - ); - await fixtures.db - .update(authSchema.session) - .set({ activeOrganizationId: org.id }) - .where(eq(authSchema.session.userId, joiner.id)); + // Stale state without raw DB writes: point the joiner's session at the + // org (if the join did not already), then the organizer removes the + // member — the removed user's session keeps its activeOrganizationId. + const joinedSession = await rpcGetSession(joinerJar); + if (joinedSession?.session?.activeOrganizationId !== org.id) { + await postBetterAuth(regressionEnv.baseUrl, "organization/set-active", { + jar: joinerJar, + body: { organizationId: org.id }, + }); + } + await removeMemberAsOrganizer(organizerJar, joiner.user.email, org.id); await page.goto(`/onboard?code=${code}`); await expect(page.getByTestId("onboard.left")).toHaveText( @@ -102,12 +133,10 @@ test.describe("onboard rescan", () => { await expect .poll(async () => { - const sessions = await fixtures.db.query.session.findMany({ - where: eq(authSchema.session.userId, joiner.id), - }); - return sessions.map((session) => session.activeOrganizationId); + const session = await rpcGetSession(joinerJar); + return session?.session?.activeOrganizationId; }) - .not.toContain(org.id); + .not.toBe(org.id); expect(pageErrors).toEqual([]); }); diff --git a/tests/regression/framework/env.go b/tests/regression/framework/env.go index 6cae4cd72..cfeaec306 100644 --- a/tests/regression/framework/env.go +++ b/tests/regression/framework/env.go @@ -65,7 +65,8 @@ func childEnv(overrides map[string]string) []string { // EMPTY. The CLI's dotenv load never overrides existing keys, so the empty // export wins over both files; the infra planner treats an empty URL as // not-supplied (zero-config dev), so no conventional database URLs are -// generated, the preflight has nothing to probe, compose stays down, and each +// generated, the preflight has nothing to probe, no infra is materialized, +// and each // stack boots on its own ephemeral pglite (plugin.dev.ts's `|| pglite` // fallback). Required keys pass through with their .env.test values. func envOverridesFromDotenv(testEnv, devEnv map[string]string) map[string]string { diff --git a/tests/regression/lib/approve-test-organizations.ts b/tests/regression/lib/approve-test-organizations.ts index 61166b7b6..6ee59fb1e 100644 --- a/tests/regression/lib/approve-test-organizations.ts +++ b/tests/regression/lib/approve-test-organizations.ts @@ -1,20 +1,28 @@ -import { inArray } from "drizzle-orm"; -import { organization } from "../../../plugins/auth/src/db/schema"; -import { createAuthTestInstance } from "./auth-test-instance"; import { computeRegressionEnv } from "./regression-env.mjs"; +/** + * HTTP org approval (ADR 0038 addendum): the stack database is in-process + * pglite, so the direct-DB seed helper is gone — approval goes through the + * admin route (`reviewOrganization`) using the boot-time bootstrap admin + * API key. Spelled the same way the Go HTTP harness invokes it: + * `node --import tsx tests/regression/lib/approve-test-organizations.ts `. + */ const ids = process.argv.slice(2); if (!ids.length) throw new Error("Organization fixture ids are required"); + const env = computeRegressionEnv(); -const fixture = await createAuthTestInstance({ - authDatabaseUrl: env.dbUrls.AUTH_DATABASE_URL, - secret: env.authSecret, -}); -try { - await fixture.db - .update(organization) - .set({ status: "active" }) - .where(inArray(organization.id, ids)); -} finally { - await fixture.close(); +for (const organizationId of ids) { + const response = await fetch(`${env.baseUrl}/api/rpc/auth/reviewOrganization`, { + method: "POST", + headers: { + "content-type": "application/json", + origin: new URL(env.baseUrl).origin, + "x-api-key": env.bootstrapAdminKey, + }, + body: JSON.stringify({ json: { organizationId, decision: "approve" } }), + }); + if (!response.ok) { + throw new Error(`approving ${organizationId}: ${response.status} ${await response.text()}`); + } } +console.log(`[approve-test-organizations] approved: ${ids.join(", ")}`); diff --git a/tests/regression/lib/auth-test-instance.ts b/tests/regression/lib/auth-test-instance.ts deleted file mode 100644 index d38512bcd..000000000 --- a/tests/regression/lib/auth-test-instance.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { betterAuth } from "better-auth"; -import { drizzleAdapter } from "better-auth/adapters/drizzle"; -import { admin, organization, testUtils } from "better-auth/plugins"; -import { drizzle } from "drizzle-orm/node-postgres"; -import { Pool } from "pg"; -import * as authSchema from "../../../plugins/auth/src/db/schema.ts"; -import { migrateTestDatabase } from "./migrate-test-db.mjs"; - -/** - * Test-only Better Auth instance for regression fixtures. - * - * Reuses the vendored auth plugin's real schema through the same - * `drizzleAdapter` wiring as the production instance, plus `testUtils()` — - * the documented Better Auth pattern for integration/E2E fixtures. Sessions - * and users minted here validate against the running server because the - * secret and auth database are shared. - */ - -export async function createAuthTestInstance({ authDatabaseUrl, secret }) { - if (!authDatabaseUrl) throw new Error("AUTH_DATABASE_URL is not configured"); - if (!secret) throw new Error("BETTER_AUTH_SECRET is not configured"); - - await migrateTestDatabase({ - migrationsDir: "plugins/auth/src/db/migrations", - databaseUrl: authDatabaseUrl, - }); - - const pool = new Pool({ connectionString: authDatabaseUrl, max: 1 }); - const db = drizzle(pool, { schema: authSchema }); - - const auth = betterAuth({ - secret, - database: drizzleAdapter(db, { - provider: "pg", - schema: authSchema, - }), - plugins: [testUtils(), organization(), admin()], - }); - const ctx = await auth.$context; - return { - auth, - test: ctx.test, - db, - close: async () => { - await pool.end(); - }, - }; -} diff --git a/tests/regression/lib/http-seed.mjs b/tests/regression/lib/http-seed.mjs new file mode 100644 index 000000000..5f124e4a3 --- /dev/null +++ b/tests/regression/lib/http-seed.mjs @@ -0,0 +1,230 @@ +import fs from "node:fs"; +import path from "node:path"; +import { computeRegressionEnv } from "./regression-env.mjs"; + +/** + * HTTP seeding for the regression stacks (ADR 0038 addendum). + * + * The stack database is in-process pglite, so fixture helpers can no longer + * write rows directly — every seed action goes through the public API surface: + * `sign-in/anonymous` mints sessions, Better Auth organization routes create + * the orgs, and two admin routes (`reviewOrganization`, `adminSetRole`) act + * with the boot-time bootstrap admin API key (AUTH_BOOTSTRAP_ADMIN_API_KEY). + * + * The artifacts match the direct-DB era byte-for-byte: `cookies.json`, + * `admin-cookies.json`, `logout-cookies.json` (Playwright cookie snaps) plus + * `admin-seed.json` / `seed.json` metadata. + */ + +const MEMBER_NAME = "member.near"; + +function uniqueSuffix() { + return `${process.pid}`; +} + +function originOf(baseUrl) { + return new URL(baseUrl).origin; +} + +function entryFromSetCookie(line, baseUrl) { + const [pair, ...attributes] = line.split(";").map((part) => part.trim()); + const separator = pair.indexOf("="); + if (separator <= 0) return null; + const name = pair.slice(0, separator); + const value = pair.slice(separator + 1); + if (!name) return null; + + const cookie = { name, value }; + let domain; + let path; + for (const attribute of attributes) { + if (/^domain=/i.test(attribute)) { + domain = attribute.slice(7).trim(); + } else if (/^path=/i.test(attribute)) { + path = attribute.slice(5).trim(); + } else if (/^secure$/i.test(attribute)) { + cookie.secure = true; + } else if (/^httponly$/i.test(attribute)) { + cookie.httpOnly = true; + } else if (/^samesite=/i.test(attribute)) { + const raw = attribute.slice(9).trim().toLowerCase(); + cookie.sameSite = raw === "none" ? "None" : raw === "strict" ? "Strict" : "Lax"; + } + } + if (domain) { + cookie.domain = domain; + } + cookie.url ??= baseUrl; + cookie.path ??= path ?? "/"; + cookie.httpOnly ??= false; + cookie.secure ??= false; + cookie.sameSite ??= "Lax"; + return cookie; +} + +/** + * Playwright `context.addCookies` entry shape — the `.json` cookie snaps the + * specs consume via `injectCookies`. + */ + +export class CookieJar { + constructor(baseUrl) { + this.baseUrl = baseUrl; + this.entries = new Map(); + } + + absorb(response) { + const lines = + typeof response.headers.getSetCookie === "function" ? response.headers.getSetCookie() : []; + for (const line of lines) { + const entry = entryFromSetCookie(line, this.baseUrl); + if (entry) this.entries.set(entry.name, entry); + } + } + + header() { + return [...this.entries.values()].map((entry) => `${entry.name}=${entry.value}`).join("; "); + } + + playwrightCookies() { + const hostname = new URL(this.baseUrl).hostname; + return [...this.entries.values()].map(({ url: _url, ...entry }) => ({ + ...entry, + domain: entry.domain ?? hostname, + })); + } +} + +export async function postBetterAuth(baseUrl, path, { jar, body, headers } = {}) { + const response = await fetch(`${baseUrl}/api/auth/${path}`, { + method: "POST", + headers: { + "content-type": "application/json", + origin: originOf(baseUrl), + ...(jar?.header() ? { cookie: jar.header() } : {}), + ...headers, + }, + ...(body ? { body: JSON.stringify(body) } : { body: JSON.stringify({}) }), + }); + if (!response.ok) { + throw new Error(`POST /api/auth/${path} failed: ${response.status} ${await response.text()}`); + } + jar?.absorb(response); + const text = await response.text(); + return text ? JSON.parse(text) : {}; +} + +export async function postRpc(env, procedure, body, extraHeaders = {}) { + const response = await fetch(`${env.baseUrl}/api/rpc/${procedure}`, { + method: "POST", + headers: { + "content-type": "application/json", + origin: originOf(env.baseUrl), + "x-api-key": env.bootstrapAdminKey, + ...extraHeaders, + }, + body: JSON.stringify({ json: body }), + }); + if (!response.ok) { + throw new Error( + `POST /api/rpc/${procedure} failed: ${response.status} ${await response.text()}`, + ); + } + return response.json(); +} + +export async function getSessionBetterAuth(baseUrl, jar) { + const response = await fetch(`${baseUrl}/api/auth/get-session`, { + headers: { + origin: originOf(baseUrl), + ...(jar?.header() ? { cookie: jar.header() } : {}), + }, + }); + if (!response.ok) { + throw new Error( + `GET /api/auth/get-session failed: ${response.status} ${await response.text()}`, + ); + } + return response.json(); +} + +export async function newAnonymousSession(env, name) { + const jar = new CookieJar(env.baseUrl); + await postBetterAuth(env.baseUrl, "sign-in/anonymous", { jar }); + await postBetterAuth(env.baseUrl, "update-user", { + jar, + body: { name }, + }); + return jar; +} + +export async function createOrganization(env, jar, name) { + const result = await postBetterAuth(env.baseUrl, "organization/create", { + jar, + body: { name, slug: name }, + headers: { origin: originOf(env.baseUrl) }, + }); + if (!result?.id) throw new Error(`organization/create returned no id: ${JSON.stringify(result)}`); + return result; +} + +export async function seedRegressionFixtures({ log = () => {} } = {}) { + const env = computeRegressionEnv(); + const unique = uniqueSuffix(); + + const adminJar = await newAnonymousSession(env, `regression-admin-${unique}`); + const logoutJar = await newAnonymousSession(env, "logout.near"); + const memberJar = await newAnonymousSession(env, MEMBER_NAME); + + const adminUser = await getSessionBetterAuth(env.baseUrl, adminJar); + const memberUser = await getSessionBetterAuth(env.baseUrl, memberJar); + if (!adminUser?.user?.id || !memberUser?.user?.id) { + throw new Error("anonymous sign-in produced no session user"); + } + + await postRpc(env, "auth/adminSetRole", { userId: adminUser.user.id, role: "admin" }); + + const adminOrgAName = `regression-admin-org-a-${unique}`; + const adminOrgBName = `regression-admin-org-b-${unique}`; + const adminOrgA = await createOrganization(env, adminJar, adminOrgAName); + const adminOrgB = await createOrganization(env, adminJar, adminOrgBName); + + const orgAName = `regression-org-a-${unique}`; + const orgBName = `regression-org-b-${unique}`; + const memberOrgA = await createOrganization(env, memberJar, orgAName); + const memberOrgB = await createOrganization(env, memberJar, orgBName); + + for (const orgId of [adminOrgA.id, adminOrgB.id, memberOrgA.id, memberOrgB.id]) { + await postRpc(env, "auth/reviewOrganization", { organizationId: orgId, decision: "approve" }); + } + + await postBetterAuth(env.baseUrl, "organization/set-active", { + jar: memberJar, + body: { organizationId: memberOrgA.id }, + }); + + const writeFile = (relativePath, value) => { + const resolved = path.resolve(process.cwd(), relativePath); + fs.mkdirSync(path.dirname(resolved), { recursive: true }); + fs.writeFileSync(resolved, JSON.stringify(value, null, 2)); + log(`[http-seed] wrote ${relativePath}`); + }; + + writeFile(".bos/regression/cookies.json", memberJar.playwrightCookies()); + writeFile(".bos/regression/admin-cookies.json", adminJar.playwrightCookies()); + writeFile(".bos/regression/logout-cookies.json", logoutJar.playwrightCookies()); + writeFile(".bos/regression/admin-seed.json", { + adminName: `regression-admin-${unique}`, + logoutName: "logout.near", + orgAName: adminOrgAName, + orgBName: adminOrgBName, + }); + writeFile(".bos/regression/seed.json", { + orgAID: memberOrgA.id, + orgBID: memberOrgB.id, + orgAName, + orgBName, + }); + + return { env, adminJar, memberJar, logoutJar }; +} diff --git a/tests/regression/lib/member-seed.ts b/tests/regression/lib/member-seed.ts deleted file mode 100644 index 3e5922a47..000000000 --- a/tests/regression/lib/member-seed.ts +++ /dev/null @@ -1,84 +0,0 @@ -import fs from "node:fs"; -import path from "node:path"; -import { eq } from "drizzle-orm"; -import * as authSchema from "../../../plugins/auth/src/db/schema.ts"; -import { createAuthTestInstance } from "./auth-test-instance.ts"; - -const MEMBER_COOKIES_PATH = ".bos/regression/cookies.json"; -const MEMBER_SEED_PATH = ".bos/regression/seed.json"; - -/** - * Seeds the member fixtures the browser regression suite consumes via - * `injectCookies`: - * - * - a member user + signed session cookie written to cookies.json - * - two owned orgs with org A active on the session - * - the matched seed.json metadata (org ids/names) - * - * Runs on every browser regression invocation (global-setup), so stale or - * wiped fixtures never reach the specs. Mirrors the Go HTTP seeder - * (tests/regression/http/seed_test.go) but goes straight to the databases - * instead of the API, so it works with whichever stack is running. - */ -export async function seedMemberFixtures({ - authDatabaseUrl, - secret, -}: { - authDatabaseUrl: string; - secret: string; -}) { - if (!authDatabaseUrl) throw new Error("AUTH_DATABASE_URL is not configured"); - if (!secret) throw new Error("BETTER_AUTH_SECRET is not configured"); - - const { test, db, close } = await createAuthTestInstance({ authDatabaseUrl, secret }); - try { - const unique = `${process.pid}`; - const member = await test.saveUser( - test.createUser({ - email: `regression-member-${unique}@regression.test`, - name: "member.near", - emailVerified: true, - }), - ); - - const orgAName = `regression-org-a-${unique}`; - const orgBName = `regression-org-b-${unique}`; - const orgA = await test.saveOrganization( - test.createOrganization({ name: orgAName, slug: orgAName }), - ); - const orgB = await test.saveOrganization( - test.createOrganization({ name: orgBName, slug: orgBName }), - ); - await test.addMember({ userId: member.id, organizationId: orgA.id, role: "owner" }); - await test.addMember({ userId: member.id, organizationId: orgB.id, role: "owner" }); - - const { session, cookies } = await test.login({ userId: member.id }); - await db - .update(authSchema.session) - .set({ activeOrganizationId: orgA.id }) - .where(eq(authSchema.session.id, session.id)); - - const cookiesResolved = path.resolve(process.cwd(), MEMBER_COOKIES_PATH); - fs.mkdirSync(path.dirname(cookiesResolved), { recursive: true }); - fs.writeFileSync(cookiesResolved, JSON.stringify(cookies, null, 2)); - console.log(`[member-seed] wrote ${cookiesResolved}`); - - const seedResolved = path.resolve(process.cwd(), MEMBER_SEED_PATH); - fs.writeFileSync( - seedResolved, - JSON.stringify( - { - orgAID: orgA.id, - orgBID: orgB.id, - orgAName, - orgBName, - }, - null, - 2, - ), - ); - console.log(`[member-seed] wrote ${seedResolved}`); - } finally { - await close(); - } -} diff --git a/tests/regression/lib/migrate-test-db.mjs b/tests/regression/lib/migrate-test-db.mjs deleted file mode 100644 index 1fda36ea7..000000000 --- a/tests/regression/lib/migrate-test-db.mjs +++ /dev/null @@ -1,149 +0,0 @@ -import { createHash } from "node:crypto"; -import { existsSync, readFileSync } from "node:fs"; -import { join, resolve } from "node:path"; -import pg from "pg"; -import { findRepoRoot } from "./regression-env.mjs"; - -const JOURNAL_SCHEMA = "drizzle"; -const JOURNAL_TABLE = "__drizzle_migrations"; - -/** - * Mirrors the plugin migrate runner (each plugin's src/db/migrate.ts): same disk - * journal layout, same sha256-of-raw-file hashes, same journal table, and the - * same preflight that records a migration as applied when its expected tables - * already exist. The regression suite needs this because Playwright's - * globalSetup runs before the webServer boots the stack, and the stack is what - * normally applies migrations at plugin init. - */ -function loadMigrationsFromDisk(migrationsDir) { - const metaDir = join(migrationsDir, "meta"); - const journalPath = join(metaDir, "_journal.json"); - - if (!existsSync(journalPath)) { - throw new Error(`Migrations journal not found at ${journalPath}. Run \`db:generate\` first.`); - } - - const journal = JSON.parse(readFileSync(journalPath, "utf8")); - - return journal.entries.map((entry) => { - const sqlPath = join(migrationsDir, `${entry.tag}.sql`); - if (!existsSync(sqlPath)) throw new Error(`Migration SQL file not found: ${sqlPath}`); - const raw = readFileSync(sqlPath, "utf8"); - return { - idx: entry.idx, - when: entry.when, - tag: entry.tag, - hash: createHash("sha256").update(raw).digest("hex"), - sql: raw - .split("--> statement-breakpoint") - .map((s) => s.trim()) - .filter(Boolean), - }; - }); -} - -function expectedTablesOf(migration) { - const tables = new Set(); - const re = /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:"([^"]+)"\.)?"([^"]+)"/gi; - for (const stmt of migration.sql) { - for (const match of stmt.matchAll(re)) { - if (match[2]) tables.add(match[2]); - } - } - return [...tables]; -} - -export async function migrateTestDatabase({ migrationsDir, databaseUrl, schemaName, repoRoot }) { - if (!databaseUrl) throw new Error("[migrate-test-db] databaseUrl is required"); - const root = repoRoot ?? findRepoRoot(); - if (!root) throw new Error("No authored config (bos.app.ts) found in any parent directory"); - - const migrations = loadMigrationsFromDisk(resolve(root, migrationsDir)); - if (migrations.length === 0) return 0; - - const client = new pg.Client({ connectionString: databaseUrl }); - await client.connect(); - try { - // One transaction under the same journal-scoped advisory lock the plugin - // runner takes — this pre-migration pass and a booting stack's own - // migration run serialize instead of colliding in pg_catalog. - await client.query("BEGIN"); - try { - await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [ - `${JOURNAL_SCHEMA}.${JOURNAL_TABLE}`, - ]); - await client.query(`CREATE SCHEMA IF NOT EXISTS "${JOURNAL_SCHEMA}"`); - await client.query(`CREATE TABLE IF NOT EXISTS "${JOURNAL_SCHEMA}"."${JOURNAL_TABLE}" ( - id SERIAL PRIMARY KEY, - hash text NOT NULL, - created_at bigint - )`); - if (schemaName) { - await client.query(`CREATE SCHEMA IF NOT EXISTS "${schemaName}"`); - } - - const appliedRows = await client.query( - `SELECT hash FROM "${JOURNAL_SCHEMA}"."${JOURNAL_TABLE}"`, - ); - const appliedHashes = new Set(appliedRows.rows.map((row) => row.hash)); - - const searchPath = schemaName - ? `SET search_path TO "${schemaName}", public` - : "SET search_path TO public"; - await client.query(searchPath); - - let applied = 0; - for (const migration of migrations) { - if (appliedHashes.has(migration.hash) || appliedHashes.has(migration.hash.slice(0, 12))) { - continue; - } - - const expectedTables = expectedTablesOf(migration); - if (expectedTables.length > 0) { - const existing = await client.query( - `SELECT table_name FROM information_schema.tables - WHERE table_schema = ANY($1::text[]) AND table_name = ANY($2::text[])`, - [schemaName ? [schemaName, "public"] : ["public"], expectedTables], - ); - const existingNames = new Set(existing.rows.map((row) => row.table_name)); - const missing = expectedTables.filter((table) => !existingNames.has(table)); - if (missing.length === 0) { - await client.query( - `INSERT INTO "${JOURNAL_SCHEMA}"."${JOURNAL_TABLE}" (hash, created_at) VALUES ($1, $2)`, - [migration.hash, migration.when], - ); - appliedHashes.add(migration.hash); - applied++; - continue; - } - } - - await client.query(`SAVEPOINT migration_${migration.idx}`); - try { - for (const statement of migration.sql) { - await client.query(statement); - } - await client.query( - `INSERT INTO "${JOURNAL_SCHEMA}"."${JOURNAL_TABLE}" (hash, created_at) VALUES ($1, $2)`, - [migration.hash, migration.when], - ); - await client.query(`RELEASE SAVEPOINT migration_${migration.idx}`); - } catch (error) { - await client.query(`ROLLBACK TO SAVEPOINT migration_${migration.idx}`); - await client.query("ROLLBACK"); - throw new Error(`[migrate-test-db] migration ${migration.tag} failed: ${error.message}`); - } - appliedHashes.add(migration.hash); - applied++; - } - - await client.query("COMMIT"); - return applied; - } catch (error) { - await client.query("ROLLBACK").catch(() => {}); - throw error; - } - } finally { - await client.end(); - } -} diff --git a/tests/regression/lib/regression-env.mjs b/tests/regression/lib/regression-env.mjs index b09c94ada..b31ffc254 100644 --- a/tests/regression/lib/regression-env.mjs +++ b/tests/regression/lib/regression-env.mjs @@ -5,9 +5,16 @@ import { fileURLToPath } from "node:url"; import { parse } from "dotenv"; const DEFAULT_BASE_URL = "http://localhost:4100"; -const DEFAULT_PG_USER = "everythingdev"; -const DEFAULT_PG_PASSWORD = "everythingdev"; const DEFAULT_BETTER_AUTH_SECRET = "regression-test-secret-do-not-use-in-production"; +/** + * ≥ 64 chars because @better-auth/api-key rejects presented keys shorter than + * the config's `defaultKeyLength` (64) before verification even runs. + */ +const DEFAULT_BOOTSTRAP_ADMIN_API_KEY = + "regression-bootstrap-admin-api-key-0123456789abcdef0123456789abcdef0123456789"; +/** The developer's persistent dev data dir default (per workspace process). */ +const DEV_PGLITE_URL = "pglite:.bos/db"; +const TEST_DATABASE_DEFAULT = "pglite::memory:"; export function findRepoRoot(startDir = process.cwd()) { let current = path.resolve(startDir); @@ -87,11 +94,26 @@ function databaseSecrets(config) { } return secrets; } -function defaultDevDatabaseUrl(secret, pgUser, pgPassword, pgHost) { - const isAuth = secret === "AUTH_DATABASE_URL"; - const port = isAuth ? 5433 : 5432; - const databaseName = isAuth ? "auth_db" : "api_db"; - return `postgres://${pgUser}:${pgPassword}@${pgHost}:${port}/${databaseName}`; +/** + * The compose-era local convention (`postgres://everythingdev:…@localhost:{5432,5433}/…` + * — plus drift ports the old allocator wrote into .env). Refused so regression + * can never reach a developer's dev data. + */ +function legacyDevDatabaseUrl(url) { + try { + const parsed = new URL(url); + if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") return false; + if (parsed.hostname !== "localhost" && parsed.hostname !== "127.0.0.1") return false; + if (parsed.username !== "everythingdev" || parsed.password !== "everythingdev") return false; + const databaseName = decodeURIComponent(parsed.pathname.replace(/^\//, "")); + return databaseName === "api_db" || databaseName === "auth_db"; + } catch { + return false; + } +} + +function isDevDatabaseUrl(url) { + return url === DEV_PGLITE_URL || legacyDevDatabaseUrl(url); } export function computeRegressionEnv({ repoRoot, env = process.env } = {}) { @@ -102,21 +124,14 @@ export function computeRegressionEnv({ repoRoot, env = process.env } = {}) { const testEnv = readDotEnv(root, ".env.test"); const allowDevDb = env.REGRESSION_ALLOW_DEV_DB === "1"; - const pgUser = env.REGRESSION_PG_USER ?? fileEnv.REGRESSION_PG_USER ?? DEFAULT_PG_USER; - const pgPassword = - env.REGRESSION_PG_PASSWORD ?? fileEnv.REGRESSION_PG_PASSWORD ?? DEFAULT_PG_PASSWORD; - const pgHost = env.REGRESSION_PG_HOST ?? fileEnv.REGRESSION_PG_HOST ?? "127.0.0.1"; - const dbUrls = {}; for (const secret of databaseSecrets(config)) { - const devUrl = fileEnv[secret] ?? defaultDevDatabaseUrl(secret, pgUser, pgPassword, pgHost); - const resolved = testEnv[secret] ?? env[secret] ?? fileEnv[secret] ?? devUrl; - if (!allowDevDb && resolved === devUrl) { + const resolved = testEnv[secret] ?? env[secret] ?? fileEnv[secret] ?? TEST_DATABASE_DEFAULT; + if (!allowDevDb && isDevDatabaseUrl(resolved)) { throw new Error( - `refusing to run: ${secret} resolves to the dev database. ` + - "Regression tests must stay isolated from dev databases. " + - "Restore the generated .env.test (run `pnpm run bos dev` to regenerate), " + - "or start the test databases with `pnpm run test:db:up`, " + + `refusing to run: ${secret} resolves to the dev database (${resolved}). ` + + "Regression tests must stay isolated from dev data. " + + "Restore the generated .env.test (in-memory pglite — run `pnpm run bos dev` to regenerate), " + "or set REGRESSION_ALLOW_DEV_DB=1 to override deliberately.", ); } @@ -137,6 +152,10 @@ export function computeRegressionEnv({ repoRoot, env = process.env } = {}) { "or set REGRESSION_ALLOW_DEV_DB=1 to override deliberately.", ); } + const bootstrapAdminKey = + testEnv.AUTH_BOOTSTRAP_ADMIN_API_KEY ?? + env.AUTH_BOOTSTRAP_ADMIN_API_KEY ?? + DEFAULT_BOOTSTRAP_ADMIN_API_KEY; const parsedBase = new URL(baseUrl); const basePort = Number(parsedBase.port) || (parsedBase.protocol === "https:" ? 443 : 80); const localPluginCount = Object.values(config.plugins ?? {}).filter( @@ -147,7 +166,15 @@ export function computeRegressionEnv({ repoRoot, env = process.env } = {}) { stalePorts.push(basePort + 10 + i); } - return { repoRoot: root, baseUrl, basePort, dbUrls, authSecret, stalePorts }; + return { + repoRoot: root, + baseUrl, + basePort, + dbUrls, + authSecret, + bootstrapAdminKey, + stalePorts, + }; } export function regressionStackOptions(config, mode, env = process.env) { @@ -217,6 +244,10 @@ export function regressionStackOptions(config, mode, env = process.env) { ...env, ...config.dbUrls, BETTER_AUTH_SECRET: config.authSecret, + // Boot seams the HTTP seeding helpers rely on (ADR 0038 addendum): + // the auth plugin instance(s) ensure a platform admin + API key so + // fixture helpers seed through the admin routes instead of the DB. + AUTH_BOOTSTRAP_ADMIN_API_KEY: config.bootstrapAdminKey, ...(mode === "dev:ssr" || mode === "backcompat" ? { BETTER_AUTH_URL: config.baseUrl } : {}), // The auth plugin's dev config derives its Better Auth baseURL from // BASE_URL — without it the instance falls back to localhost:3000 and @@ -246,6 +277,7 @@ if (isDirectRun) { baseUrl: result.baseUrl, dbUrls: result.dbUrls, authSecret: result.authSecret, + bootstrapAdminKey: result.bootstrapAdminKey, stalePorts: result.stalePorts, }, null, diff --git a/tests/regression/lib/regression-env.test.mjs b/tests/regression/lib/regression-env.test.mjs index c950aa548..1acf51422 100644 --- a/tests/regression/lib/regression-env.test.mjs +++ b/tests/regression/lib/regression-env.test.mjs @@ -78,14 +78,40 @@ void test("ambient test settings reach the stack when no test file exists", () = assert.equal(stack.env.BETTER_AUTH_SECRET, env.BETTER_AUTH_SECRET); }); -void test("dev database rejection does not reveal credentials", () => { +void test("dev secret rejection does not reveal credentials", () => { const repoRoot = fixture(); assert.throws( () => computeRegressionEnv({ repoRoot, env: {} }), - (error) => error.message.includes("dev database") && !error.message.includes("private"), + (error) => error.message.startsWith("refusing to run") && !error.message.includes("private"), ); }); +void test("dev pglite data dir rejection names the secret and never reveals data", () => { + const repoRoot = fixture(); + writeFileSync( + join(repoRoot, ".env"), + "API_DATABASE_URL=pglite:.bos/db\nBETTER_AUTH_SECRET=private-dev\n", + ); + assert.throws( + () => computeRegressionEnv({ repoRoot, env: {} }), + (error) => + error.message.includes("API_DATABASE_URL") && + error.message.includes("dev database") && + !error.message.includes("private-dev"), + ); +}); + +void test("in-memory pglite and external postgres URLs are accepted test targets", () => { + const repoRoot = fixture(); + writeFileSync( + join(repoRoot, ".env.test"), + "API_DATABASE_URL=pglite::memory:\nBETTER_AUTH_SECRET=test-secret\n", + ); + const config = computeRegressionEnv({ repoRoot, env: {} }); + assert.equal(config.dbUrls.API_DATABASE_URL, "pglite::memory:"); + assert.equal(config.dbUrls.AUTH_DATABASE_URL, "pglite::memory:"); +}); + void test("invalid modes fail before starting a stack", () => { assert.throws(() => regressionStackOptions({}, "invalid", {}), /unknown mode/); }); diff --git a/tests/regression/lib/start-container.mjs b/tests/regression/lib/start-container.mjs index 3ac62c3db..4134b8cf6 100644 --- a/tests/regression/lib/start-container.mjs +++ b/tests/regression/lib/start-container.mjs @@ -42,6 +42,14 @@ const run = (cmd, args, options = {}) => const waitForDatabases = async () => { const deadline = Date.now() + 60_000; for (const url of Object.values(regressionEnv.dbUrls)) { + // PGlite URLs (`.bos/db` data dirs, `:memory:`) are in-process — the + // container's services boot their own database, nothing to probe here. + try { + const protocol = new URL(url).protocol; + if (protocol !== "postgres:" && protocol !== "postgresql:") continue; + } catch { + // malformed URL — the stack will surface it + } const parsed = new URL(url); let reachable = false; while (Date.now() < deadline && !reachable) { @@ -61,8 +69,9 @@ const waitForDatabases = async () => { } }; -// The test databases live on the runner; the container reaches them through -// the docker host gateway. +// Postgres URLs pointed at the runner reach the container through the docker +// host gateway; pglite URLs pass through untouched (the container boots its +// own in-process database). const containerDbUrls = Object.fromEntries( Object.entries(regressionEnv.dbUrls).map(([key, url]) => [ key, @@ -73,6 +82,9 @@ const containerDbUrls = Object.fromEntries( const childEnv = { ...containerDbUrls, BETTER_AUTH_SECRET: regressionEnv.authSecret, + // The auth plugin's boot-time seeding seam — the HTTP helpers approve orgs + // and set roles through the admin routes (ADR 0038 addendum). + AUTH_BOOTSTRAP_ADMIN_API_KEY: regressionEnv.bootstrapAdminKey, BASE_URL: regressionEnv.baseUrl, CORS_ORIGIN: regressionEnv.baseUrl, REGRESSION_VARIANT: variant, diff --git a/tests/regression/lib/start-stack.mjs b/tests/regression/lib/start-stack.mjs index ddb75ad9e..3435eb991 100644 --- a/tests/regression/lib/start-stack.mjs +++ b/tests/regression/lib/start-stack.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node // Boots the regression stack (dev/prod/backcompat) with the test environment -// from .env.test so regression runs never touch dev databases or dev ports. +// from .env.test (in-memory pglite) so regression runs never touch dev data +// or dev ports. import { spawn } from "node:child_process"; import { mkdirSync, openSync } from "node:fs"; import net from "node:net"; @@ -55,7 +56,7 @@ async function waitForDatabases(dbUrls) { if (attempt === PROBE_RETRIES) { throw new Error( `test databases not reachable: ${missing.join(", ")}. ` + - "Start them with `pnpm run test:db:up` (docker compose postgres-*-test services).", + "Set each *_DATABASE_URL to a reachable postgres:// URL (or leave .env.test on the in-memory pglite default).", ); } log(`waiting for test databases (${missing.join(", ")})...`);