diff --git a/.env.compose.example b/.env.compose.example index dd55b89..83ef694 100644 --- a/.env.compose.example +++ b/.env.compose.example @@ -3,9 +3,8 @@ # apps/web/.env.example and apps/indexer/.env.example. # --- Passwords (local development values; Compose composes the connection URLs) --- -# Only the administrator password is an input. The three runtime roles -# (xcs_indexer, xcs_api and the dedicated pg_monitor identity xcs_monitor) are -# created by db/compose-initdb/01-runtime-roles.sql when the postgres volume is +# Only the administrator password is an input. All eight runtime users are +# created by db/compose-initdb/01-runtime-roles.sql when the PostgreSQL volume is # first initialised, with published non-secret local-development passwords that # docker-compose.yml uses verbatim. In a deployment those users are created by # the managed database service instead; db:bootstrap only assigns privileges and @@ -62,10 +61,9 @@ XCS_PINNING_IP_HASH_SECRET= # file named by XCS_NETWORK_PROFILE above, so there is nothing to set here. # Public, browser-reachable endpoint. Never put credentials or a private indexer URL here. XCS_PUBLIC_RPC_URL=wss://testnet.xrpl-labs.com/ -# Optional Xaman OAuth configuration. The API key is public. Register the exact redirect URL -# in the Xaman Developer Console; it must be this deployment's origin with a trailing slash. +# Optional Xaman OAuth configuration. The API key is public. Register the site origin +# in the Xaman Developer Console; the wallet SDK owns the redirect callback. XCS_PUBLIC_XAMAN_API_KEY= -XCS_PUBLIC_XAMAN_REDIRECT_URL=http://localhost:3000/ XCS_PUBLIC_WALLET_CONNECT_PROJECT_ID= # --- Host bindings --- @@ -114,3 +112,18 @@ XCS_GRAFANA_MEMORY_LIMIT=512m XCS_GRAFANA_CPU_LIMIT=0.5 XCS_EXPORTER_MEMORY_LIMIT=128m XCS_EXPORTER_CPU_LIMIT=0.25 + + +# --- Optional application overlay --- +# Providing this complete group enables auth, Commons review and issuer journeys. +XCS_AUTH_ORIGIN=https://localhost:3445 +XCS_IDENTITY_CLIENT_ID= +XCS_IDENTITY_CLIENT_SECRET= +XCS_ADMIN_DOCUMENT_KEY= +XCS_MAILPIT_PORT=8025 +XCS_NOTIFIER_IMAGE=xcs-notifier:local + +# --- Optional hosted-payload overlay --- +XCS_PUBLIC_PAYLOAD_BASE_URL= +XCS_PAYLOAD_STORAGE_IP_HASH_SECRET= +XCS_HOSTED_PAYLOAD_NETWORKS= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22aa9ed..61ee5a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,10 +20,6 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # The vendored-copy check reads a retired source (the former - # packages/db) with `git show :`, which the default - # depth-1 clone does not contain. - fetch-depth: 0 - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 with: version: 10.34.4 @@ -73,6 +69,8 @@ jobs: # the production (security) configuration. - name: Run browser pilot flows run: pnpm --dir apps/web test:e2e + - name: Run authentication browser flows + run: pnpm --dir apps/web test:e2e:auth - name: Run browser security flows run: pnpm --dir apps/web test:e2e:security - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 @@ -99,8 +97,22 @@ jobs: --health-interval 5s --health-timeout 3s --health-retries 20 + mailpit: + image: axllent/mailpit:v1.29.2 + ports: + - 1025:1025 + - 8025:8025 + options: >- + --health-cmd "/mailpit readyz" + --health-interval 5s + --health-timeout 3s + --health-retries 20 env: XCS_TEST_DATABASE_URL: postgres://postgres:postgres-integration-only-password-0001@127.0.0.1:5432/postgres + XCS_TEST_MAILPIT_URL: http://127.0.0.1:8025 + XCS_TEST_SMTP_PORT: '1025' + XCS_ISSUER_RUNTIME_MAILPIT_ORIGIN: http://127.0.0.1:8025 + XCS_ISSUER_RUNTIME_SMTP_PORT: '1025' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -122,7 +134,31 @@ jobs: # it runs against this job's service rather than a second database. - run: pnpm --dir apps/web install --ignore-workspace --frozen-lockfile - run: pnpm --dir apps/web test:postgres - - name: Verify generated database baseline is committed + - name: Install Chromium for compiled authenticated flows + run: pnpm --dir apps/web exec playwright install --with-deps chromium + - name: Exercise compiled Nitro with restricted PostgreSQL pools + run: pnpm test:runtime + - name: Verify generated database migrations are committed run: | pnpm --dir apps/indexer db:generate git diff --exit-code -- db/migrations + test -z "$(git ls-files --others --exclude-standard -- db/migrations)" + + deployment-images: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + component: [web, indexer, notifier, db-bootstrap] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Build ${{ matrix.component }} deployment image + run: >- + docker build + --file apps/${{ matrix.component }}/Dockerfile + --tag xcs-${{ matrix.component }}:ci + --build-arg XCS_REVISION=${{ github.sha }} + . diff --git a/.github/workflows/operations.yml b/.github/workflows/operations.yml index 0325d05..245da5a 100644 --- a/.github/workflows/operations.yml +++ b/.github/workflows/operations.yml @@ -15,19 +15,17 @@ jobs: env: COMPOSE_PROJECT_NAME: xcs-ci-${{ github.run_id }}-${{ github.run_attempt }} XCS_POSTGRES_ADMIN_PASSWORD: compose-ci-admin-password-00000001 - # The three runtime roles are created by db/compose-initdb/01-runtime-roles.sql - # with published local-development passwords; they are not inputs. + XCS_AUTH_ORIGIN: https://compose.invalid + XCS_IDENTITY_CLIENT_ID: compose-ci-client + XCS_IDENTITY_CLIENT_SECRET: compose-ci-identity-secret-000001 + XCS_ADMIN_DOCUMENT_KEY: compose-ci-document-signing-key-000001 XCS_METRICS_TOKEN: compose-ci-metrics-token-000000000001 XCS_RPC_URL_PRIMARY: wss://primary.invalid.example XCS_RPC_URL_SECONDARY: wss://secondary.invalid.example - # XCS_NETWORK_PROFILE is deliberately unset. The only committed profile, - # config/networks/testnet.example.json, is an invalid template, and both - # services now read this file: the indexer (not started here) would refuse - # it at preflight and the web service, which derives the browser-visible - # profile identifier from it, would refuse to start. XCS_PUBLIC_RPC_URL: wss://submission.invalid.example XCS_INDEXER_IMAGE: xcs-indexer:ci XCS_WEB_IMAGE: xcs-web:ci + XCS_NOTIFIER_IMAGE: xcs-notifier:ci steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -49,48 +47,8 @@ jobs: chmod 644 ops/secrets/xcs_metrics_token test -s ops/secrets/xcs_metrics_token - - name: Render and validate every Compose profile - shell: bash - run: | - set -euo pipefail - docker compose config --quiet - docker compose --profile monitoring --profile demo-pinning config --quiet - docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile monitoring config --quiet - - mapfile -t DEFAULT_SERVICES < <(docker compose config --services | sort) - EXPECTED=(db-bootstrap indexer postgres web) - if [[ "${DEFAULT_SERVICES[*]}" != "${EXPECTED[*]}" ]]; then - printf 'Unexpected default services: %s\n' "${DEFAULT_SERVICES[*]}" >&2 - exit 1 - fi - - docker compose --profile monitoring --profile demo-pinning config --format json \ - > "$RUNNER_TEMP/xcs-compose.json" - jq -e ' - (.services.postgres.ports == null) and - (.services.prometheus.ports == null) and - (.services.grafana.ports == null) and - (.services["postgres-exporter"].ports == null) and - (.services["node-exporter"].ports == null) and - (.services.ipfs.ports == null) and - (.services.prometheus.profiles | index("monitoring") != null) and - (.services.grafana.profiles | index("monitoring") != null) and - (.networks.database.internal == true) and - (.networks.monitoring.internal == true) and - ([.services[] | select(.image | endswith(":latest"))] | length == 0) - ' "$RUNNER_TEMP/xcs-compose.json" - - docker compose -f docker-compose.yml -f docker-compose.dev.yml \ - --profile monitoring --profile demo-pinning config --format json \ - > "$RUNNER_TEMP/xcs-compose-dev.json" - jq -e ' - [ - .services.postgres.ports[], - .services.prometheus.ports[], - .services.grafana.ports[], - .services.ipfs.ports[] - ] | all(.host_ip == "127.0.0.1") - ' "$RUNNER_TEMP/xcs-compose-dev.json" + - name: Render and validate every Compose overlay and profile + run: node ops/ci/check-compose-overlays.mjs - name: Validate monitoring configuration with Prometheus shell: bash @@ -108,13 +66,13 @@ jobs: jq empty ops/monitoring/grafana/dashboards/*.json - name: Build production-only core images - run: docker compose build web indexer + run: docker compose -f docker-compose.yml -f docker-compose.application.yml build web indexer admin-notifier - name: Assert runtime images exclude development tools shell: bash run: | set -euo pipefail - for IMAGE in "$XCS_WEB_IMAGE" "$XCS_INDEXER_IMAGE"; do + for IMAGE in "$XCS_WEB_IMAGE" "$XCS_INDEXER_IMAGE" "$XCS_NOTIFIER_IMAGE"; do docker run --rm --entrypoint sh "$IMAGE" -c \ 'test ! -e node_modules/typescript && test ! -e node_modules/vitest && test ! -e node_modules/tsx' done @@ -157,39 +115,30 @@ jobs: fi # The bootstrap is deliberately restart-safe. Run it again after the - # dependency chain has completed to prove repeated initialization and - # role provisioning do not require a clean volume. + # dependency chain has completed to prove repeated migrations and + # privilege grants do not require a clean volume. "${COMPOSE[@]}" run --rm --no-deps db-bootstrap curl --fail --silent --show-error http://127.0.0.1:3000/health/live > /dev/null READY_STATUS="$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:3000/health/ready)" test "$READY_STATUS" = '503' - # The metrics routes are enabled by the presence of a valid - # XCS_METRICS_TOKEN, which this job supplies to the web service, so they - # are registered here and must be gated rather than absent: an - # unauthenticated request is rejected and the job's own token is - # accepted. The routes deliberately stay 200 while the database is - # unavailable, which is the state this stack is in. - for METRICS_PATH in /internal/metrics /internal/metrics/prometheus; do - METRICS_STATUS="$(curl --silent --output /dev/null --write-out '%{http_code}' \ - "http://127.0.0.1:3000${METRICS_PATH}")" - test "$METRICS_STATUS" = '401' - WRONG_TOKEN_STATUS="$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --header 'Authorization: Bearer compose-ci-wrong-token-0000000000001' \ - "http://127.0.0.1:3000${METRICS_PATH}")" - test "$WRONG_TOKEN_STATUS" = '401' - AUTHENTICATED_STATUS="$(curl --silent --output /dev/null --write-out '%{http_code}' \ - --header "Authorization: Bearer $XCS_METRICS_TOKEN" \ - "http://127.0.0.1:3000${METRICS_PATH}")" - test "$AUTHENTICATED_STATUS" = '200' - done + UNAUTHENTICATED_STATUS="$(curl --silent --output /dev/null --write-out '%{http_code}' http://127.0.0.1:3000/internal/metrics/prometheus)" + test "$UNAUTHENTICATED_STATUS" = '401' + curl --fail --silent --show-error \ + --header "Authorization: Bearer $XCS_METRICS_TOKEN" \ + http://127.0.0.1:3000/internal/metrics/prometheus \ + | grep --fixed-strings 'xcs_database_available' if "${COMPOSE[@]}" ps --services --status running | grep --extended-regexp '^(prometheus|grafana|postgres-exporter|node-exporter|ipfs)$'; then echo 'A non-core profile service started unexpectedly' >&2 exit 1 fi - if PUBLISHED_POSTGRES="$("${COMPOSE[@]}" port postgres 5432 2>/dev/null)" && [[ -n "$PUBLISHED_POSTGRES" ]]; then - echo "PostgreSQL was unexpectedly published at $PUBLISHED_POSTGRES" >&2 + POSTGRES_CONTAINER="$("${COMPOSE[@]}" ps --quiet postgres)" + # Compose versions can report "invalid IP:0" for an exposed but + # unpublished port. Inspect the actual Engine bindings instead. + if ! docker inspect --format '{{json .NetworkSettings.Ports}}' "$POSTGRES_CONTAINER" \ + | jq --exit-status '[.[]? | select(. != null) | .[]?] | length == 0' > /dev/null; then + echo 'PostgreSQL unexpectedly has published host ports' >&2 exit 1 fi @@ -197,11 +146,14 @@ jobs: --command " SELECT pg_has_role('xcs_monitor', 'pg_monitor', 'MEMBER') - AND (SELECT bool_and(rolcanlogin) FROM pg_roles - WHERE rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor')) - AND (SELECT rolconnlimit FROM pg_roles WHERE rolname = 'xcs_indexer') = 12 - AND (SELECT rolconnlimit FROM pg_roles WHERE rolname = 'xcs_api') = 12 - AND (SELECT rolconnlimit FROM pg_roles WHERE rolname = 'xcs_monitor') = 3 + AND NOT EXISTS ( + SELECT 1 FROM pg_roles + WHERE rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + AND ( + NOT rolcanlogin OR rolsuper OR rolcreaterole OR rolcreatedb + OR rolreplication OR rolbypassrls + ) + ) AND NOT has_schema_privilege('xcs_monitor', 'public', 'CREATE') AND NOT EXISTS ( SELECT 1 @@ -231,6 +183,35 @@ jobs: ); " | grep --fixed-strings --line-regexp t + "${COMPOSE[@]}" exec -T postgres psql -U xcs_admin -d xcs --no-psqlrc --tuples-only --no-align \ + --command " + SELECT + has_table_privilege('xcs_payload_writer', 'hosted_payloads', 'INSERT') + AND has_table_privilege('xcs_issuer', 'app_issuer_payloads', 'INSERT') + AND has_table_privilege('xcs_app', 'app_sessions', 'INSERT') + AND has_column_privilege('xcs_admin_app', 'app_organization_applications', 'status', 'UPDATE') + AND has_column_privilege('xcs_notifier', 'app_admin_notifications', 'status', 'UPDATE') + AND NOT EXISTS ( + SELECT 1 FROM pg_roles + WHERE rolname IN ('xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer', 'xcs_payload_writer') + AND ( + NOT rolcanlogin OR rolsuper OR rolcreaterole OR rolcreatedb + OR rolreplication OR rolbypassrls + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM pg_class relation + JOIN pg_namespace namespace ON namespace.oid = relation.relnamespace + CROSS JOIN (VALUES ('xcs_api'), ('xcs_indexer'), ('xcs_payload_writer')) AS runtime(role_name) + WHERE namespace.nspname = 'public' AND relation.relname LIKE 'app_%' + AND relation.relkind IN ('r', 'p') + AND has_table_privilege(runtime.role_name, relation.oid, 'SELECT,INSERT,UPDATE,DELETE') + ) + AND NOT has_table_privilege('xcs_admin_app', 'app_issuer_payloads', 'SELECT') + AND NOT has_table_privilege('xcs_notifier', 'app_issuer_payloads', 'SELECT') + AND NOT has_table_privilege('xcs_app', 'app_issuer_payloads', 'SELECT'); + " | grep --fixed-strings --line-regexp t + "${COMPOSE[@]}" --profile monitoring up --no-build --detach --wait postgres-exporter prometheus PROMETHEUS_CONTAINER="$("${COMPOSE[@]}" --profile monitoring ps --quiet prometheus)" PROMETHEUS_STATE="$(docker inspect --format '{{.State.Status}}' "$PROMETHEUS_CONTAINER")" @@ -274,7 +255,7 @@ jobs: shell: bash run: | set -euo pipefail - for DOCKERFILE in apps/web/Dockerfile apps/indexer/Dockerfile; do + for DOCKERFILE in apps/web/Dockerfile apps/indexer/Dockerfile apps/notifier/Dockerfile apps/db-bootstrap/Dockerfile; do grep --fixed-strings -- 'USER node' "$DOCKERFILE" grep --fixed-strings -- 'pnpm install --frozen-lockfile' "$DOCKERFILE" if grep --extended-regexp 'RUN --mount|<<[A-Z]' "$DOCKERFILE"; then diff --git a/.gitignore b/.gitignore index 9b4924e..b4872e9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ coverage/ .nuxt/ .output/ .data/ +.do/ .env .env.* !.env.compose.example diff --git a/AGENTS.md b/AGENTS.md index 1ec1af1..d3695c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,11 +1,18 @@ # Repository Codex Instructions +- Keep active source worktrees, wallet-test browser profiles, unpacked wallet extensions, and long-lived Docker bind mounts in durable directories, never OS temporary directories. Before opening a wallet test site, verify the extension files exist and select the browser profile where it is installed. +- In browser wallet checks, resolve local certificate warnings before testing provider availability. A failed CDP navigation may expose an internal error document, not the visible browser interstitial; do not assume its URL or DOM is the site or disable certificate verification globally. +- On macOS, Chromium ignores hostname-scoped keychain trust rules. Verify local TLS in the target browser as well as with native SecTrust; never remove the hostname restriction or broaden CA trust without explicit user approval. +- Before reading paths from an earlier architecture, another XCS checkout, or another agent’s planned file list, resolve them with `rg --files` in the active worktree; layouts differ between worktrees and concurrent implementations can choose different final paths. Resolve directory arguments too before narrowing a search to a presumed tooling or scripts directory. Reuse discovered filenames verbatim; use absolute paths for files belonging to an outer or sibling checkout. - The Compose stack is local development only. Before pushing changes to it, render the exact overlays with every referenced profile and, when a Docker engine is available, build `apps/web/Dockerfile` and `apps/indexer/Dockerfile` with the repository root as build context and start the stack under the `node` runtime user; configuration rendering alone does not validate the build context or the shared `db/` and `config/` copies the images depend on. - Every per-app pnpm command that resolves dependencies — `install`, `audit`, `licenses list` — must carry `--ignore-workspace`. Without it, `pnpm --dir apps/web install` silently operates on the root workspace and still exits 0. - When cryptographically checking an XRPL transaction returned by `tx`, request `binary: true` and verify the canonical `tx_blob`; API v2 `tx_json` can rename serialized fields (for example, `Amount` is exposed as `DeliverMax`) and is not the canonical object to re-encode for signature verification. -- For Playwright smoke assertions, inspect the rendered semantics and use a unique role, test ID, or explicitly scoped locator; do not assume localized headings or repeated public identifiers resolve to exactly one element in strict mode. +- For Playwright smoke assertions, inspect the rendered semantics and use a unique role, test ID, or explicitly scoped locator; scope navigation links to the named main-navigation region because the footer repeats them. Read the current locale catalog after concurrent UI edits. Do not assume localized headings or repeated public identifiers resolve to exactly one element in strict mode. Inspect the accessibility snapshot before matching unfamiliar controls: concatenated `textContent` and CSS-uppercase visible text do not determine an exact accessible name. After changing labels or disclosure controls, search both `e2e/` and compiled browser scenarios in `test/` for affected selectors; default unit runs skip those runtime scenarios. - When running a second Nuxt dev or Playwright web-server instance from another worktree, set `NUXT_IGNORE_LOCK=1` and use a distinct port; Nuxt's development lock is shared across worktrees. - After editing Prettier-managed files, format the touched files before running format checks; do not use the check command as the formatter. +- For scripted text replacements, assert that the expected original text exists and inspect the resulting diff before rerunning validation; a successful script exit does not prove that an edit occurred. - For one-off Node scripts, resolve dependencies from the owning workspace package; running from the monorepo root can silently select an incompatible ancestor `node_modules` package. - The two apps do not import `@xcs-protocol/*`. Protocol behaviour changes land in `packages/core` first and are mirrored by hand into `apps/web/app/lib/xcs/` and `apps/indexer/src/lib/xcs/` in the same pull request; rebuild the owning app, not the package, to validate an app change. - Quote shell paths that contain route brackets or other glob metacharacters before passing them to commands such as `sed` or `rg`. +- For wallet connection regressions, derive SignIn responses, live queries and initialization events separately from the installed extension’s public code, then test connection, account refresh and pre-signature checks through the real SDK. Treat account/network notifications as a request to recheck identity, not proof that an approved session changed. +- On macOS, keep the wallet test browser within the visible screen and foreground it before requesting approval. If GemWallet rejects without a popup, inspect its service worker for window-bounds errors before diagnosing a permission or signing failure. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3425b08..3cd145f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,13 +44,13 @@ The rule is therefore: The copy locations are: -| Copy | Source | -| ---------------------------- | ------------------------------------------------- | -| `apps/web/app/lib/xcs/core/` | `packages/core/src/**` | -| `apps/web/app/lib/xcs/sdk/` | `packages/sdk/src/**` | -| `apps/web/server/lib/db/` | the former `packages/db` client and transactions | -| `apps/indexer/src/lib/xcs/` | the `packages/core` modules the indexer reaches | -| `apps/indexer/src/lib/db/` | client, fencing, transactions, provision, migrate | +| Copy | Source | +| ---------------------------- | ----------------------------------------------- | +| `apps/web/app/lib/xcs/core/` | `packages/core/src/**` | +| `apps/web/app/lib/xcs/sdk/` | `packages/sdk/src/**` | +| `apps/web/server/lib/db/` | application-local database implementation | +| `apps/indexer/src/lib/xcs/` | the `packages/core` modules the indexer reaches | +| `apps/indexer/src/lib/db/` | application-local database implementation | ### How the rule is enforced @@ -73,10 +73,11 @@ from` module specifier — which is the case this check exists for, since a chan - a new file under a vendored directory with no header at all, so a copy cannot be added unlabelled; - a stale by-design divergence (below). -It compares against the source **as it is now**, not as it was at the recorded ``. The sha is -immutable, so comparing against it would only ever notice edits to the copy. A source that has been -retired from the tree (the former `packages/db`) can no longer change, so for those files the -recorded commit is used. +It compares protocol copies against the source **as it is now**, not as it was at the recorded +``. The sha is immutable, so comparing against it would only ever notice edits to the copy. The +former `packages/db` implementation was retired before this repository history and cannot be +resolved here; its descendants are explicitly marked as application-local code and are covered by +each application's tests. Shared tables and migrations remain canonical under `db/`. A copy that genuinely has to differ — because the surrounding application requires it, not because someone edited it — declares that on a second header line, pinning the exact source the divergence diff --git a/README.md b/README.md index 702dac6..e64f8f9 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,16 @@ This repository is alpha software for XRPL Testnet. Do not use personal data or ## Layout -Two deployable applications: +Two application runtimes and two one-purpose deployment components: - `apps/indexer`: validated-ledger ingestion and rebuildable projections. It owns the database tooling (`db:generate`, `db:migrate`, `db:bootstrap`). - `apps/web`: the Nuxt Testnet explorer and issuer/subject workflows **and** the read/verification API it serves from the same origin at `/v1`, with `/health/*`, `/internal/metrics*` and `/documentation`. +- `apps/notifier`: the restricted SMTP notification worker built from the web application. +- `apps/db-bootstrap`: the pre-deploy migration and least-privilege role provisioning job built from + the indexer. Each application is standalone: its own `package.json`, `pnpm-lock.yaml`, `.npmrc`, tsconfig, Prettier config, `Dockerfile` and `.env.example`. Neither imports a workspace package; each carries @@ -76,11 +79,10 @@ pnpm --dir apps/indexer verify ### Pointing the applications at a database -PostgreSQL is provisioned outside this repository, and so are its users: create `xcs_indexer`, -`xcs_api` and `xcs_monitor` in the DigitalOcean control panel or with `doctl databases user create` -and keep the passwords DigitalOcean generates. The indexer then owns the grants; run it once against -a fresh database as the administrator, then give each application its least-privilege connection -string: +PostgreSQL is provisioned outside this repository. First create these eight managed users in +DigitalOcean: `xcs_indexer`, `xcs_api`, `xcs_payload_writer`, `xcs_monitor`, `xcs_app`, +`xcs_admin_app`, `xcs_notifier` and `xcs_issuer`. DigitalOcean generates and owns their passwords. +Then run the grants-only bootstrap with the database administrator URL: ```bash XCS_BOOTSTRAP_DATABASE_URL=postgres://xcs_admin:…@host:5432/xcs \ @@ -88,10 +90,13 @@ XCS_BOOTSTRAP_DATABASE_URL=postgres://xcs_admin:…@host:5432/xcs \ pnpm --dir apps/indexer db:bootstrap ``` -The indexer then reads `XCS_INDEXER_DATABASE_URL` (role `xcs_indexer`) and the web app reads -`XCS_DATABASE_URL` (role `xcs_api`). The complete contracts are +The command applies migrations 0000–0008 and database grants atomically. It never creates a user and +never reads, sets or rotates a runtime password. Each component then receives its own managed URL. +The complete contracts are [`apps/indexer/.env.example`](./apps/indexer/.env.example) and -[`apps/web/.env.example`](./apps/web/.env.example). +[`apps/web/.env.example`](./apps/web/.env.example). The pre-deploy and notification contracts are +[`apps/db-bootstrap/.env.example`](./apps/db-bootstrap/.env.example) and +[`apps/notifier/.env.example`](./apps/notifier/.env.example). For a disposable local database, `docker-compose.yml` runs PostgreSQL, the one-shot bootstrap and both applications. It is a **local-development stack only**; it is not a deployment template. Copy diff --git a/apps/db-bootstrap/.env.example b/apps/db-bootstrap/.env.example new file mode 100644 index 0000000..ee85950 --- /dev/null +++ b/apps/db-bootstrap/.env.example @@ -0,0 +1,4 @@ +# One-off pre-deploy contract. DigitalOcean creates all eight runtime users and +# owns their generated passwords; this job never receives or changes them. +XCS_BOOTSTRAP_DATABASE_URL= # held (managed PostgreSQL administrator) +XCS_DATABASE_CLUSTER_SCOPE=dedicated # required acknowledgement diff --git a/apps/db-bootstrap/Dockerfile b/apps/db-bootstrap/Dockerfile new file mode 100644 index 0000000..5fe5fb2 --- /dev/null +++ b/apps/db-bootstrap/Dockerfile @@ -0,0 +1,39 @@ +# One-shot database migration and role bootstrap job. Build context: repository root. +ARG XCS_NODE_IMAGE=node:24-alpine +FROM ${XCS_NODE_IMAGE} AS build + +RUN npm install -g pnpm@10.34.4 + +WORKDIR /workspace +COPY apps/indexer ./apps/indexer +COPY db ./db +COPY config ./config + +WORKDIR /workspace/apps/indexer +RUN pnpm install --frozen-lockfile +RUN pnpm build +RUN pnpm install --frozen-lockfile --prod + +FROM ${XCS_NODE_IMAGE} AS runtime + +ARG XCS_REVISION=unknown +ARG XCS_VERSION=0.0.0 + +LABEL org.opencontainers.image.title="XCS database bootstrap" \ + org.opencontainers.image.description="XCS pre-deploy migrations and least-privilege roles" \ + org.opencontainers.image.source="https://github.com/XRPL-Commons/XCS" \ + org.opencontainers.image.revision="${XCS_REVISION}" \ + org.opencontainers.image.version="${XCS_VERSION}" \ + org.opencontainers.image.licenses="MIT" + +ENV NODE_ENV=production +ENV XCS_MIGRATIONS_DIR=/workspace/db/migrations + +WORKDIR /workspace +COPY --from=build --chown=node:node /workspace/apps/indexer/dist ./dist +COPY --from=build --chown=node:node /workspace/apps/indexer/node_modules ./node_modules +COPY --from=build --chown=node:node /workspace/apps/indexer/package.json ./package.json +COPY --chown=node:node db/migrations ./db/migrations + +USER node +CMD ["node", "dist/lib/db/bin/bootstrap.js"] diff --git a/apps/indexer/.env.example b/apps/indexer/.env.example index c6fb361..4db10ec 100644 --- a/apps/indexer/.env.example +++ b/apps/indexer/.env.example @@ -1,47 +1,15 @@ -# Contract for the XCS indexer worker. -# Names only; values live in the deployment's secret store. -# -# This file holds secrets and values that genuinely differ per deployment. -# Behavioural settings -- the poll interval, the lease duration and the batch -# size -- are checked-in code, in src/settings.ts; they are not deployment -# inputs and a variable of the same name here has no effect. - -# PostgreSQL connection string for the indexer role. -XCS_INDEXER_DATABASE_URL= # config -# Commons-operated complete-history rippled endpoint, supplied out of band. +# Runtime worker contract. Secrets and deployment-specific values are supplied by +# the deployment platform; worker behaviour defaults live in src/settings.ts. +XCS_INDEXER_DATABASE_URL= # held XCS_RPC_URL_PRIMARY= # held # Public fallback endpoint; no availability or retention SLA for XCS. XCS_RPC_URL_SECONDARY= # held -# Path to the network profile JSON this worker indexes; must be the same -# published file the web app serves. Every profile is its own operator-created -# file (config/networks/README.md), so this one cannot live in code. +# Path to the same published network profile JSON served by the web app. XCS_NETWORK_PROFILE= # config -# The three values below default in src/settings.ts to the safe public posture -# (`blackholed`, `shared`, and no acknowledgement). They stay overridable because -# ADR 0003's disposable controlled Testnet pilot is the one deployment that needs -# the weaker registry posture, and it must still supply every acknowledgement -# explicitly. Never set them for a public profile. -# `blackholed` or `controlled-testnet-pilot` (ADR 0003 only). +# Safe defaults live in src/settings.ts. These three overrides exist only for the +# disposable controlled Testnet pilot described by ADR 0003. XCS_REGISTRY_POLICY= # optional config -# Must equal DISPOSABLE_PROFILE_AND_DATABASE for controlled-testnet-pilot. XCS_CONTROLLED_PILOT_ACK= # optional config -# `shared` or `exclusive-profile`; the controlled pilot requires exclusive-profile. XCS_DATABASE_SCOPE= # optional config - -# The two values below are read only by db:bootstrap, run once from an operator -# machine or a one-off job against the external database. The long-running -# service reads only the variables above. -# -# db:bootstrap is grants-only. The managed database service owns the users: create -# xcs_indexer, xcs_api and xcs_monitor in the DigitalOcean control panel or with -# `doctl databases user create`, take the passwords DigitalOcean generates, and -# only then run db:bootstrap as the administrator. It never creates a role and -# never sets, resets or reads a role password; a missing role fails the run by -# name. No runtime password is an input here. -XCS_BOOTSTRAP_DATABASE_URL= # optional held -# Must be exactly `dedicated`: an acknowledgement, not a choice. The grants and -# role attributes db:bootstrap applies are cluster-visible, so it refuses to run -# on a cluster the operator does not own. -XCS_DATABASE_CLUSTER_SCOPE= # optional config diff --git a/apps/indexer/README.md b/apps/indexer/README.md index 1c4cab6..c8d4c4b 100644 --- a/apps/indexer/README.md +++ b/apps/indexer/README.md @@ -26,8 +26,10 @@ pnpm --dir apps/indexer dev # tsx watch src/main.ts pnpm --dir apps/indexer verify # format:check, lint, test, build ``` -Configuration is the contract in [`.env.example`](./.env.example). The long-running service reads the -first ten variables; the last two are read only by `db:bootstrap`. +Configuration for the long-running service is the contract in [`.env.example`](./.env.example). +Bootstrap secrets have a separate contract in +[`apps/db-bootstrap/.env.example`](../db-bootstrap/.env.example) and must never be delivered to the +indexer worker. The committed `config/networks/testnet.example.json` is a placeholder with an invalid registry and activation boundary. The indexer refuses it and stops with `SOURCE_REGISTRY_NOT_BLACKHOLED`. **This is @@ -40,18 +42,22 @@ The indexer owns every database command. `drizzle-kit` and the migration runner devDependencies; the web app has none of them. ```sh -pnpm --dir apps/indexer db:generate # regenerate db/migrations after editing db/schema — commit the result +pnpm --dir apps/indexer db:generate # add a migration after editing db/schema — retain applied files pnpm --dir apps/indexer db:migrate # apply migrations to an existing database (idempotent) -pnpm --dir apps/indexer db:bootstrap # migrate + grant privileges to xcs_indexer / xcs_api / xcs_monitor +pnpm --dir apps/indexer db:bootstrap # migrate + grant all eight runtime users ``` -`db:bootstrap` runs once against a fresh database and needs only `XCS_BOOTSTRAP_DATABASE_URL` and -`XCS_DATABASE_CLUSTER_SCOPE=dedicated`. It is **grants-only**: the managed database service owns the -users, so create `xcs_indexer`, `xcs_api` and `xcs_monitor` in the DigitalOcean control panel or with -`doctl databases user create` first, take the passwords DigitalOcean generates, then run this once as -the administrator. It never creates a role and never sets, resets or reads a role password; if a role -is missing it fails and names it. It is idempotent. Rotating a runtime password is now a DigitalOcean -operation and does not need this step. PostgreSQL is provisioned outside this repository; see +Before `db:bootstrap`, create all eight runtime users listed in [`db/README.md`](../../db/README.md) +through DigitalOcean and retain the generated connection strings in the deployment secret store. +The bootstrap needs only `XCS_BOOTSTRAP_DATABASE_URL` and +`XCS_DATABASE_CLUSTER_SCOPE=dedicated`. It checks every user before DDL, applies migrations 0000–0008 +and least-privilege grants in one transaction, and never reads or changes a runtime password. + +`db:migrate` applies pending migrations without provisioning roles. The committed 0000–0008 history +supports populated-baseline upgrades and repeated runs; integration tests check preservation of +profiles and legacy payload bytes/locators. Never rewrite an applied migration. This is not an +upgrader for arbitrary schema drift or the former Nuxt MVP. See the [shared database contract](../../db/README.md). +PostgreSQL is provisioned outside this repository; see [`docs/database.md`](../../docs/database.md) and the [deployment runbook](../../docs/runbooks/deployment.md). @@ -71,6 +77,22 @@ the selected registry policy on both sources, and prints no endpoint or credenti procedures — healthy state, recovery, deterministic rebuild and evidence capture — are in the [indexer runbook](../../docs/runbooks/indexer.md). +Ledger transport requests the complete transaction set with `transactions: true`, `expand: true` +and `binary: true`. Large expanded JSON responses can exceed a provider's WebSocket limit even +when that provider retains the ledger. The official binary codec decodes the header, transaction +bytes and metadata locally; XRPL hash helpers validate the header and derive transaction IDs. +Protocol pseudo-transactions use the standard unsigned transaction-ID domain. Canonical serialized +fields such as `Payment.Amount` are preserved instead of API v2 JSON aliases such as `DeliverMax`. +See the [XRPL ledger API](https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/ledger-methods/ledger). + +Both independently operated sources must still agree on the complete normalized header and every +transaction/metadata object. Malformed blobs, conflicting hashes, missing transactions and a source +without the required history remain fatal. The transport change neither skips checkpoints nor +resets a projection or reduces the quorum. `ripple-binary-codec@2.11.0` is a direct dependency for +its official ledger-header decoder, matching the version already used transitively by `xrpl`. +An existing database with a different migration journal still requires its own reviewed upgrade; +rebuilding the indexer image does not authorize rewriting that history. + ## Tests ```sh diff --git a/apps/indexer/package.json b/apps/indexer/package.json index aaede45..491e563 100644 --- a/apps/indexer/package.json +++ b/apps/indexer/package.json @@ -14,7 +14,7 @@ "ceremony": "tsx src/ceremony-cli.ts", "clean": "rm -rf dist coverage", "db:bootstrap": "tsx src/lib/db/bin/bootstrap.ts", - "db:generate": "cd ../../db && drizzle-kit generate --config ./drizzle.config.ts", + "db:generate": "NODE_PATH=./node_modules drizzle-kit generate --config ../../db/drizzle.config.ts", "db:migrate": "tsx src/lib/db/bin/migrate.ts", "dev": "tsx watch src/main.ts", "fixture:capture": "tsx src/fixture-cli.ts capture", @@ -28,9 +28,11 @@ "start": "node dist/main.js", "test": "vitest run", "test:coverage": "vitest run --coverage", - "test:postgres": "XCS_REQUIRE_POSTGRES_TESTS=1 vitest run test/postgres.integration.test.ts", + "test:postgres": "XCS_REQUIRE_POSTGRES_TESTS=1 vitest run --no-file-parallelism test/db/migrations.integration.test.ts test/db/app-model.integration.test.ts test/db/auth.integration.test.ts test/db/admin.integration.test.ts test/postgres.integration.test.ts", "typecheck": "tsc -p tsconfig.json --noEmit", - "verify": "pnpm format:check && pnpm lint && pnpm test && pnpm build" + "verify": "pnpm format:check && pnpm lint && pnpm test && pnpm build", + "db:status": "tsx src/lib/db/bin/status.ts", + "admin:bootstrap": "tsx src/lib/db/bin/admin-bootstrap.ts" }, "dependencies": { "@noble/hashes": "2.3.0", @@ -40,6 +42,7 @@ "jsonc-parser": "3.3.1", "multiformats": "14.0.5", "postgres": "^3.4.7", + "ripple-binary-codec": "2.11.0", "tr46": "4.1.1", "xrpl": "^5.0.0" }, diff --git a/apps/indexer/pnpm-lock.yaml b/apps/indexer/pnpm-lock.yaml index bc35c47..61c58bd 100644 --- a/apps/indexer/pnpm-lock.yaml +++ b/apps/indexer/pnpm-lock.yaml @@ -33,6 +33,9 @@ importers: postgres: specifier: ^3.4.7 version: 3.4.9 + ripple-binary-codec: + specifier: 2.11.0 + version: 2.11.0 tr46: specifier: 4.1.1 version: 4.1.1 diff --git a/apps/indexer/src/lib/db/app/bootstrap-admin.ts b/apps/indexer/src/lib/db/app/bootstrap-admin.ts new file mode 100644 index 0000000..dd6f98c --- /dev/null +++ b/apps/indexer/src/lib/db/app/bootstrap-admin.ts @@ -0,0 +1,37 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import type { DatabaseClient } from '../client.js' + +/** Explicit one-time operator action. Never invoked by login or migration. */ +export async function bootstrapFirstAdmin( + client: DatabaseClient, + input: { issuer: string; subject: string; operator: string }, +): Promise<{ userId: string; created: boolean }> { + if ( + !input.issuer.trim() || + !input.subject.trim() || + !input.operator.trim() || + input.operator.length > 200 + ) { + throw new Error('ADMIN_BOOTSTRAP_INPUT_INVALID') + } + return client.sql.begin(async (sql) => { + await sql`SELECT pg_advisory_xact_lock(1480807217, 30)` + const [user] = await sql<{ id: string }[]>`SELECT id FROM app_users + WHERE identity_issuer = ${input.issuer} AND identity_subject = ${input.subject} AND status = 'active' FOR UPDATE` + if (!user) throw new Error('ADMIN_BOOTSTRAP_ACCOUNT_NOT_FOUND') + const roles = await sql< + { user_id: string; revoked_at: Date | null }[] + >`SELECT user_id, revoked_at FROM app_user_roles WHERE role = 'admin' FOR UPDATE` + if (roles.some((role) => role.user_id === user.id && role.revoked_at !== null)) + throw new Error('ADMIN_BOOTSTRAP_REVOKED') + if (roles.some((role) => role.user_id !== user.id && role.revoked_at === null)) + throw new Error('ADMIN_BOOTSTRAP_ALREADY_INITIALIZED') + if (roles.some((role) => role.user_id === user.id)) return { userId: user.id, created: false } + // Historical bootstrap also prevents using this command to replace a removed administrator. + const previous = await sql`SELECT id FROM app_admin_bootstrap_audit LIMIT 1` + if (previous.length) throw new Error('ADMIN_BOOTSTRAP_ALREADY_INITIALIZED') + await sql`INSERT INTO app_user_roles (user_id, role, granted_by) VALUES (${user.id}, 'admin', ${user.id})` + await sql`INSERT INTO app_admin_bootstrap_audit (user_id, operator) VALUES (${user.id}, ${input.operator.trim()})` + return { userId: user.id, created: true } + }) +} diff --git a/apps/indexer/src/lib/db/app/invitations.ts b/apps/indexer/src/lib/db/app/invitations.ts new file mode 100644 index 0000000..d4e5fd1 --- /dev/null +++ b/apps/indexer/src/lib/db/app/invitations.ts @@ -0,0 +1,41 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { and, eq, exists, gt, isNull, sql } from 'drizzle-orm' + +import type { XcsDatabase } from '../client.js' +import { appInvites, appUsers } from '#db/schema/app/index.js' +import { hashAppToken } from './tokens.js' + +/** Call only from an authenticated mutation, with userId supplied by the server's session. */ +export async function claimInvitation(db: XcsDatabase, input: { token: string; userId: string }) { + const tokenHash = hashAppToken(input.token) + if (tokenHash === null) return null + + // One conditional UPDATE is the claim boundary. Statement time, not transaction start, + // prevents a caller's long-running transaction from extending the invitation's lifetime. + const [claimed] = await db + .update(appInvites) + .set({ claimedBy: input.userId, claimedAt: sql`statement_timestamp()` }) + .where( + and( + eq(appInvites.tokenHash, tokenHash), + isNull(appInvites.claimedAt), + isNull(appInvites.revokedAt), + gt(appInvites.expiresAt, sql`statement_timestamp()`), + exists( + db + .select({ id: appUsers.id }) + .from(appUsers) + .where(and(eq(appUsers.id, input.userId), eq(appUsers.status, 'active'))), + ), + ), + ) + .returning({ + id: appInvites.id, + organizationId: appInvites.organizationId, + profileId: appInvites.profileId, + schemaUid: appInvites.schemaUid, + claimedBy: appInvites.claimedBy, + claimedAt: appInvites.claimedAt, + }) + return claimed ?? null +} diff --git a/apps/indexer/src/lib/db/app/tokens.ts b/apps/indexer/src/lib/db/app/tokens.ts new file mode 100644 index 0000000..3505451 --- /dev/null +++ b/apps/indexer/src/lib/db/app/tokens.ts @@ -0,0 +1,13 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { createHash, randomBytes } from 'node:crypto' + +/** Bearer secrets stay outside the database; persist only the returned hash. */ +export function createAppToken(): { token: string; tokenHash: string } { + const token = randomBytes(32).toString('base64url') + return { token, tokenHash: hashAppToken(token)! } +} + +export function hashAppToken(token: string): string | null { + if (!/^[A-Za-z0-9_-]{43}$/.test(token)) return null + return createHash('sha256').update(token, 'utf8').digest('hex') +} diff --git a/apps/indexer/src/lib/db/app/visibility.ts b/apps/indexer/src/lib/db/app/visibility.ts new file mode 100644 index 0000000..f89729d --- /dev/null +++ b/apps/indexer/src/lib/db/app/visibility.ts @@ -0,0 +1,169 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { and, eq, isNull, sql } from 'drizzle-orm' +import { alias } from 'drizzle-orm/pg-core' + +import type { XcsDatabase } from '../client.js' +import { + appCredentialMetadata, + appOrganizationApplications, + appOrganizations, + appPresentations, + appUsers, +} from '#db/schema/app/index.js' +import { hashAppToken } from './tokens.js' + +export interface CredentialAccess { + scope: 'public' | 'full' + publicFields: string[] +} + +/** All authorization facts are read together, never accepted as request-supplied roles/approvals. */ +export async function getCredentialAccess( + db: XcsDatabase, + input: { + profileId: string + generationId: string + viewerUserId: string | null + presentationToken?: string + }, +): Promise { + const issuer = alias(appOrganizations, 'issuer_organization') + const verifier = alias(appOrganizations, 'verifier_organization') + const tokenHash = + input.presentationToken === undefined ? null : hashAppToken(input.presentationToken) + + const [row] = await db + .select({ + visibility: appCredentialMetadata.visibility, + publicFields: appCredentialMetadata.publicFields, + viewerId: appUsers.id, + recipientUserId: appCredentialMetadata.recipientUserId, + issuerResponsibleUserId: issuer.responsibleUserId, + issuerStatus: issuer.status, + verifierResponsibleUserId: verifier.responsibleUserId, + verifierStatus: verifier.status, + verifierApproval: appOrganizationApplications.status, + }) + .from(appCredentialMetadata) + .leftJoin( + appUsers, + input.viewerUserId === null + ? sql`FALSE` + : and(eq(appUsers.id, input.viewerUserId), eq(appUsers.status, 'active')), + ) + .leftJoin(issuer, eq(issuer.id, appCredentialMetadata.issuerOrganizationId)) + .leftJoin( + appPresentations, + tokenHash === null + ? sql`FALSE` + : and( + eq(appPresentations.tokenHash, tokenHash), + eq(appPresentations.profileId, appCredentialMetadata.profileId), + eq(appPresentations.generationId, appCredentialMetadata.generationId), + eq(appPresentations.recipientUserId, appCredentialMetadata.recipientUserId), + eq(appPresentations.scope, 'full'), + isNull(appPresentations.revokedAt), + ), + ) + .leftJoin(verifier, eq(verifier.id, appPresentations.verifierOrganizationId)) + .leftJoin( + appOrganizationApplications, + and( + eq(appOrganizationApplications.organizationId, verifier.id), + eq(appOrganizationApplications.role, 'verifier'), + ), + ) + .where( + and( + eq(appCredentialMetadata.profileId, input.profileId), + eq(appCredentialMetadata.generationId, input.generationId), + ), + ) + if (!row) return null + + const isOwner = + row.viewerId !== null && + (row.viewerId === row.recipientUserId || + (row.viewerId === row.issuerResponsibleUserId && row.issuerStatus === 'active')) + const isAuthorizedVerifier = + row.viewerId !== null && + row.viewerId === row.verifierResponsibleUserId && + row.verifierStatus === 'active' && + row.verifierApproval === 'approved' + return { + scope: row.visibility === 'public' || isOwner || isAuthorizedVerifier ? 'full' : 'public', + publicFields: row.publicFields, + } +} + +export type ClaimValue = + null | boolean | number | string | ClaimValue[] | { [key: string]: ClaimValue } +export type Claims = { [key: string]: ClaimValue } + +interface Selection { + whole: boolean + children: Map +} + +/** RFC 6901 pointers relative to claims. Objects can be narrowed; arrays are atomic fields. */ +export function validatePublicFields(paths: readonly string[]): void { + selection(paths) +} + +function selection(paths: readonly string[]): Selection { + if (!Array.isArray(paths) || paths.length > 256) throw new Error('INVALID_PUBLIC_FIELDS') + const root: Selection = { whole: false, children: new Map() } + for (const path of paths) { + if ( + typeof path !== 'string' || + !path.startsWith('/') || + path.length > 1024 || + /~(?![01])/u.test(path) + ) { + throw new Error('INVALID_PUBLIC_FIELD_PATH') + } + const parts = path + .slice(1) + .split('/') + .map((part) => part.replaceAll('~1', '/').replaceAll('~0', '~')) + if ( + parts.length > 32 || + parts.some((part) => ['__proto__', 'constructor', 'prototype'].includes(part)) + ) { + throw new Error('INVALID_PUBLIC_FIELD_PATH') + } + let node = root + for (const part of parts) { + let child = node.children.get(part) + if (!child) { + child = { whole: false, children: new Map() } + node.children.set(part, child) + } + node = child + } + node.whole = true + } + return root +} + +function project(source: Claims, node: Selection): Claims { + const result: Claims = {} + for (const [key, child] of node.children) { + if (!Object.hasOwn(source, key)) continue + const value = source[key]! + if (child.whole) { + result[key] = structuredClone(value) + } else if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + const nested = project(value, child) + if (Object.keys(nested).length > 0) result[key] = nested + } + } + return result +} + +/** Return only claims, never spread a private payload envelope into a public response. */ +export function filterCredentialClaims(claims: Claims, access: CredentialAccess): Claims { + if (access.scope === 'full') return structuredClone(claims) + if (access.scope !== 'public') throw new Error('INVALID_CREDENTIAL_ACCESS') + return project(claims, selection(access.publicFields)) +} diff --git a/apps/indexer/src/lib/db/bin/admin-bootstrap.ts b/apps/indexer/src/lib/db/bin/admin-bootstrap.ts new file mode 100644 index 0000000..7cf646b --- /dev/null +++ b/apps/indexer/src/lib/db/bin/admin-bootstrap.ts @@ -0,0 +1,27 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { createDatabaseClient } from '../client.js' +import { bootstrapFirstAdmin } from '../app/bootstrap-admin.js' +import { requiredEnvironment } from './environment.js' + +try { + const client = createDatabaseClient(requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL'), { + onNotice: () => undefined, + }) + try { + const result = await bootstrapFirstAdmin(client, { + issuer: requiredEnvironment('XCS_ADMIN_IDENTITY_ISSUER'), + subject: requiredEnvironment('XCS_ADMIN_IDENTITY_SUBJECT'), + operator: requiredEnvironment('XCS_ADMIN_BOOTSTRAP_OPERATOR'), + }) + process.stdout.write(`${JSON.stringify({ ok: true, created: result.created })}\n`) + } finally { + await client.close() + } +} catch (error) { + const code = + error instanceof Error && /^ADMIN_BOOTSTRAP_[A-Z_]+$/.test(error.message) + ? error.message + : 'ADMIN_BOOTSTRAP_FAILED' + process.stderr.write(`${JSON.stringify({ ok: false, code })}\n`) + process.exitCode = 1 +} diff --git a/apps/indexer/src/lib/db/bin/bootstrap.ts b/apps/indexer/src/lib/db/bin/bootstrap.ts index e3fa9e2..8a39d1d 100644 --- a/apps/indexer/src/lib/db/bin/bootstrap.ts +++ b/apps/indexer/src/lib/db/bin/bootstrap.ts @@ -1,29 +1,20 @@ -// Copied from packages/db/src/bin/bootstrap.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). -// Diverges by design (provisioning is grants-only, so no runtime password is read from the environment, and a missing role is reported by name); source sha256:d9142aa3fda17e3032510d9dd0405a01464b531e59ed59299d5823c06e2fa8fd. +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import { bootstrapDatabase, DatabaseBootstrapConfigurationError, MissingRuntimeDatabaseRolesError, parseDatabaseClusterScope, UnsafeRuntimeDatabaseRolesError, + XCS_RUNTIME_DATABASE_ROLES, } from '../bootstrap.js' import { createDatabaseClient } from '../client.js' - -function requiredEnvironment(name: string): string { - const value = process.env[name] - if (value === undefined || value.trim().length === 0) { - throw new DatabaseBootstrapConfigurationError( - `${name} is required. Set it in the same command that runs this step: a bare ` + - 'shell assignment on its own line is a shell variable, not an environment ' + - 'variable, and this process never sees it.', - ) - } - return value -} +import { requiredEnvironment } from './environment.js' async function main(): Promise { const databaseUrl = requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL') - const client = createDatabaseClient(databaseUrl) + const client = createDatabaseClient(databaseUrl, { + onNotice: () => undefined, + }) try { const report = await bootstrapDatabase(client, { @@ -35,7 +26,7 @@ async function main(): Promise { process.stdout.write( `${JSON.stringify({ ok: true, - roles: ['xcs_indexer', 'xcs_api', 'xcs_monitor'], + roles: XCS_RUNTIME_DATABASE_ROLES, administrator: report.administrator, unappliedResourceControls: report.unappliedResourceControls, })}\n`, @@ -66,7 +57,12 @@ try { ? { roles: error.roles, findings: error.findings } : {} process.stderr.write( - `${JSON.stringify({ ok: false, code: error.code, ...roles, message: error.message })}\n`, + `${JSON.stringify({ + ok: false, + code: error.code, + ...roles, + message: error.message, + })}\n`, ) } else { // Drivers and migration helpers wrap the original failure, so walk the diff --git a/apps/indexer/src/lib/db/bin/environment.ts b/apps/indexer/src/lib/db/bin/environment.ts new file mode 100644 index 0000000..77df208 --- /dev/null +++ b/apps/indexer/src/lib/db/bin/environment.ts @@ -0,0 +1,36 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { readFileSync, statSync } from 'node:fs' + +import { DatabaseBootstrapConfigurationError } from '../provision.js' + +/** Supports secret files in native CLI runs as well as the container entrypoint. */ +export function requiredEnvironment(name: string): string { + const direct = process.env[name] + const path = process.env[`${name}_FILE`] + if (direct && path) { + throw new DatabaseBootstrapConfigurationError( + `DATABASE_ENVIRONMENT_CONFLICT: ${name} and ${name}_FILE cannot both be set`, + ) + } + let value = direct + if (path) { + const stat = statSync(path) + if (!stat.isFile() || stat.size < 1 || stat.size > 16_384) { + throw new DatabaseBootstrapConfigurationError( + `DATABASE_SECRET_FILE_INVALID: ${name}_FILE must contain one non-empty value of at most 16384 bytes`, + ) + } + value = readFileSync(path, 'utf8').replace(/\r?\n$/, '') + if (/[\r\n]/.test(value)) { + throw new DatabaseBootstrapConfigurationError( + `DATABASE_SECRET_FILE_INVALID: ${name}_FILE must contain exactly one line`, + ) + } + } + if (!value?.trim()) { + throw new DatabaseBootstrapConfigurationError( + `DATABASE_ENVIRONMENT_REQUIRED: ${name} is required. Set it in the same command that runs this step; a bare shell assignment on its own line is not exported to the process.`, + ) + } + return value +} diff --git a/apps/indexer/src/lib/db/bin/migrate.ts b/apps/indexer/src/lib/db/bin/migrate.ts index b46dcd9..3c208d7 100644 --- a/apps/indexer/src/lib/db/bin/migrate.ts +++ b/apps/indexer/src/lib/db/bin/migrate.ts @@ -1,33 +1,20 @@ -// Not a vendored copy (no upstream: packages/db never had a bin/migrate.ts): applies -// db/migrations without provisioning roles. -import { initializeDatabase, migrationsFolder } from '../bootstrap.js' +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import { createDatabaseClient } from '../client.js' +import { migrateDatabase } from '../migrations.js' +import { requiredEnvironment } from './environment.js' -function administratorDatabaseUrl(): string { - for (const name of ['XCS_BOOTSTRAP_DATABASE_URL', 'XCS_DATABASE_URL'] as const) { - const value = process.env[name] - if (value !== undefined && value.trim().length > 0) return value - } - throw new Error('XCS_BOOTSTRAP_DATABASE_URL or XCS_DATABASE_URL is required') -} - -async function main(): Promise { - const client = createDatabaseClient(administratorDatabaseUrl()) - +try { + const client = createDatabaseClient(requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL'), { + onNotice: () => undefined, + }) try { - // The migrator records applied migrations in `drizzle.__drizzle_migrations`, - // so a second run against the same database applies nothing. - await initializeDatabase(client) - process.stdout.write(`${JSON.stringify({ ok: true, migrationsFolder: migrationsFolder() })}\n`) + const result = await migrateDatabase(client) + process.stdout.write(`${JSON.stringify({ ok: true, ...result })}\n`) } finally { await client.close() } -} - -try { - await main() } catch { - // Do not serialize the thrown error: connection errors may contain a URL or password. - process.stderr.write(`${JSON.stringify({ ok: false, code: 'DATABASE_MIGRATE_FAILED' })}\n`) + // Database errors may include connection credentials or SQL; never serialize them. + process.stderr.write(`${JSON.stringify({ ok: false, code: 'DATABASE_MIGRATION_FAILED' })}\n`) process.exitCode = 1 } diff --git a/apps/indexer/src/lib/db/bin/status.ts b/apps/indexer/src/lib/db/bin/status.ts new file mode 100644 index 0000000..f194e1c --- /dev/null +++ b/apps/indexer/src/lib/db/bin/status.ts @@ -0,0 +1,21 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +import { createDatabaseClient } from '../client.js' +import { databaseMigrationStatus } from '../migrations.js' +import { requiredEnvironment } from './environment.js' + +try { + const client = createDatabaseClient(requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL'), { + onNotice: () => undefined, + }) + try { + const result = await databaseMigrationStatus(client) + process.stdout.write(`${JSON.stringify({ ok: true, ...result })}\n`) + } finally { + await client.close() + } +} catch { + process.stderr.write( + `${JSON.stringify({ ok: false, code: 'DATABASE_MIGRATION_STATUS_FAILED' })}\n`, + ) + process.exitCode = 1 +} diff --git a/apps/indexer/src/lib/db/bootstrap.ts b/apps/indexer/src/lib/db/bootstrap.ts index 378d6da..a8dbe4d 100644 --- a/apps/indexer/src/lib/db/bootstrap.ts +++ b/apps/indexer/src/lib/db/bootstrap.ts @@ -1,33 +1,27 @@ -// Copied from packages/db/src/bootstrap.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). -// Diverges by design (migrations moved from packages/db/drizzle to db/migrations, with the XCS_MIGRATIONS_DIR override the container image sets, and provisioning is grants-only so no password travels through this module); source sha256:ef5dc1f95861fc8465a9d68e59e25f5a55eedab91a483b9d34ec89abf7346ca3. -import { fileURLToPath } from 'node:url' - -import { migrate } from 'drizzle-orm/postgres-js/migrator' - +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import type { DatabaseClient } from './client.js' +import { migrateDatabase, migrateDatabaseInTransaction } from './migrations.js' import { - provisionRuntimeDatabasePrivileges, + applyRuntimeDatabasePrivilegesInTransaction, + parseDatabaseClusterScope, + prepareRuntimeDatabaseProvisioningInTransaction, type RuntimeDatabaseProvisioning, type RuntimeDatabaseProvisioningReport, } from './provision.js' -const DEFAULT_MIGRATIONS_FOLDER = fileURLToPath( - new URL('../../../../../db/migrations', import.meta.url), -) - -export function migrationsFolder(): string { - const override = process.env.XCS_MIGRATIONS_DIR?.trim() - return override === undefined || override.length === 0 ? DEFAULT_MIGRATIONS_FOLDER : override -} - export { DatabaseBootstrapConfigurationError, MissingRuntimeDatabaseRolesError, + UnsafeRuntimeDatabaseRolesError, parseDatabaseClusterScope, provisionRuntimeDatabasePrivileges, - UnsafeRuntimeDatabaseRolesError, XCS_API_DATABASE_CONNECTION_LIMIT, XCS_API_DATABASE_ROLE, + XCS_APP_DATABASE_ROLE, + XCS_ADMIN_APP_DATABASE_ROLE, + XCS_NOTIFIER_DATABASE_ROLE, + XCS_ISSUER_DATABASE_ROLE, + XCS_PAYLOAD_WRITER_DATABASE_ROLE, XCS_DATABASE_CLUSTER_SCOPE, XCS_INDEXER_DATABASE_CONNECTION_LIMIT, XCS_INDEXER_DATABASE_ROLE, @@ -40,13 +34,22 @@ export { } from './provision.js' export async function initializeDatabase(client: DatabaseClient): Promise { - await migrate(client.db, { migrationsFolder: migrationsFolder() }) + await migrateDatabase(client) } export async function bootstrapDatabase( client: DatabaseClient, provisioning: RuntimeDatabaseProvisioning, ): Promise { - await initializeDatabase(client) - return await provisionRuntimeDatabasePrivileges(client, provisioning) + // Reject an invalid acknowledgement before reserving a connection. + parseDatabaseClusterScope(provisioning.clusterScope) + return await client.sql.begin(async (transaction) => { + // DigitalOcean must have created every user before any migration changes state. + const administrator = await prepareRuntimeDatabaseProvisioningInTransaction( + transaction, + provisioning, + ) + await migrateDatabaseInTransaction(transaction) + return await applyRuntimeDatabasePrivilegesInTransaction(transaction, administrator) + }) } diff --git a/apps/indexer/src/lib/db/client.ts b/apps/indexer/src/lib/db/client.ts index bd0a28c..fa20878 100644 --- a/apps/indexer/src/lib/db/client.ts +++ b/apps/indexer/src/lib/db/client.ts @@ -1,8 +1,8 @@ -// Copied from packages/db/src/client.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' import postgres, { type Sql } from 'postgres' -import * as schema from '#db/schema' +import * as schema from '#db/schema/index.js' export type XcsDatabase = PostgresJsDatabase @@ -12,7 +12,10 @@ export interface DatabaseClient { close: () => Promise } -export function createDatabaseClient(databaseUrl: string): DatabaseClient { +export function createDatabaseClient( + databaseUrl: string, + options: { onNotice?: () => void } = {}, +): DatabaseClient { if (databaseUrl.trim().length === 0) { throw new Error('DATABASE_URL must not be empty') } @@ -22,6 +25,7 @@ export function createDatabaseClient(databaseUrl: string): DatabaseClient { idle_timeout: 20, connect_timeout: 10, prepare: false, + ...(options.onNotice ? { onnotice: options.onNotice } : {}), }) return { diff --git a/apps/indexer/src/lib/db/index.ts b/apps/indexer/src/lib/db/index.ts new file mode 100644 index 0000000..345e16c --- /dev/null +++ b/apps/indexer/src/lib/db/index.ts @@ -0,0 +1,10 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +export * from './client.js' +export * from './indexer-fencing.js' +export * from './transactions.js' +export * from '#db/schema/index.js' +export * from '#db/schema/app/index.js' +export * from './app/tokens.js' +export * from './app/invitations.js' +export * from './app/visibility.js' +export * from './app/bootstrap-admin.js' diff --git a/apps/indexer/src/lib/db/indexer-fencing.ts b/apps/indexer/src/lib/db/indexer-fencing.ts index 73e53fe..f7ed0d3 100644 --- a/apps/indexer/src/lib/db/indexer-fencing.ts +++ b/apps/indexer/src/lib/db/indexer-fencing.ts @@ -1,4 +1,4 @@ -// Copied from packages/db/src/indexer-fencing.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import { and, eq, gt, isNull, lte, or, sql } from 'drizzle-orm' import type { XcsDatabase } from './client.js' @@ -8,7 +8,7 @@ import { indexerStatuses, type IndexerStatusRow, type IndexerStatusState, -} from '#db/schema' +} from '#db/schema/index.js' const UINT32_MAX = 4_294_967_295 const MAX_SAFE_EPOCH = Number.MAX_SAFE_INTEGER diff --git a/apps/indexer/src/lib/db/migrations.ts b/apps/indexer/src/lib/db/migrations.ts new file mode 100644 index 0000000..ea631a4 --- /dev/null +++ b/apps/indexer/src/lib/db/migrations.ts @@ -0,0 +1,160 @@ +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +// Diverges by design (shared db/migrations location and XCS_MIGRATIONS_DIR deployment override); source sha256:42c7d5380bba58dcfc4414b033d5e9c028d5dc93bec2ddce6db860d43cb1abae. +import { existsSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { readMigrationFiles } from 'drizzle-orm/migrator' +import type { Sql, TransactionSql } from 'postgres' + +import type { DatabaseClient } from './client.js' + +const DEFAULT_MIGRATIONS_FOLDERS = [ + fileURLToPath(new URL('../../../../../db/migrations', import.meta.url)), + fileURLToPath(new URL('../../../db/migrations', import.meta.url)), +] + +export function migrationsFolder(): string { + return ( + process.env.XCS_MIGRATIONS_DIR?.trim() || + DEFAULT_MIGRATIONS_FOLDERS.find((folder) => existsSync(join(folder, 'meta/_journal.json'))) || + DEFAULT_MIGRATIONS_FOLDERS[0]! + ) +} + +export const DATABASE_MIGRATIONS_FOLDER = migrationsFolder() +const LOCK_CLASS = 1_480_807_217 +const LOCK_OBJECT = 2 + +export interface MigrationStatus { + applied: string[] + pending: string[] +} + +interface MigrationEntry { + tag: string + when: number + hash: string + statements: string[] +} + +function readCatalog(folder: string): MigrationEntry[] { + const journal = JSON.parse(readFileSync(join(folder, 'meta/_journal.json'), 'utf8')) as { + dialect?: unknown + entries?: Array<{ idx?: unknown; tag?: unknown; when?: unknown }> + } + if ( + journal.dialect !== 'postgresql' || + !Array.isArray(journal.entries) || + journal.entries.length === 0 + ) { + throw new Error('DATABASE_MIGRATION_CATALOG_INVALID') + } + const tags = new Set() + let previous = 0 + for (const [index, entry] of journal.entries.entries()) { + if ( + entry.idx !== index || + typeof entry.tag !== 'string' || + !/^\d{4}_[a-zA-Z0-9_-]+$/.test(entry.tag) || + tags.has(entry.tag) || + typeof entry.when !== 'number' || + !Number.isSafeInteger(entry.when) || + entry.when <= previous + ) { + throw new Error('DATABASE_MIGRATION_CATALOG_INVALID') + } + tags.add(entry.tag) + previous = entry.when + } + const migrations = readMigrationFiles({ migrationsFolder: folder }) + return migrations.map((migration, index) => ({ + tag: journal.entries![index]!.tag as string, + when: migration.folderMillis, + hash: migration.hash, + statements: migration.sql, + })) +} + +async function status( + sql: Sql | TransactionSql, + catalog: MigrationEntry[], +): Promise { + const [table] = await sql<{ exists: boolean }[]>` + SELECT to_regclass('drizzle.__drizzle_migrations') IS NOT NULL AS exists + ` + // Inspection of an empty database must not create the Drizzle schema or journal. + const applied = table?.exists + ? await sql<{ hash: string; created_at: string | number | null }[]>` + SELECT hash, created_at FROM drizzle.__drizzle_migrations ORDER BY created_at, id + ` + : [] + if ( + applied.length > catalog.length || + applied.some((row, index) => { + const expected = catalog[index]! + return row.hash !== expected.hash || String(row.created_at) !== String(expected.when) + }) + ) { + throw new Error('DATABASE_MIGRATION_HISTORY_MISMATCH') + } + return { + applied: catalog.slice(0, applied.length).map((entry) => entry.tag), + pending: catalog.slice(applied.length).map((entry) => entry.tag), + } +} + +export async function databaseMigrationStatus( + client: DatabaseClient, + migrationsFolder = DATABASE_MIGRATIONS_FOLDER, +): Promise { + return status(client.sql, readCatalog(migrationsFolder)) +} + +/** Applies the existing Drizzle journal without rotating passwords or widening runtime grants. */ +export async function migrateDatabase( + client: DatabaseClient, + migrationsFolder = DATABASE_MIGRATIONS_FOLDER, +): Promise { + const catalog = readCatalog(migrationsFolder) + return client.sql.begin((transaction) => migrateCatalogInTransaction(transaction, catalog)) +} + +async function migrateCatalogInTransaction( + transaction: TransactionSql, + catalog: MigrationEntry[], +): Promise { + // The stock migrator reads history before its transaction. Read, validate and apply on + // one locked transaction instead, retaining Drizzle's SQL reader and exact journal format. + const [lock] = await transaction<{ locked: boolean }[]>` + SELECT pg_try_advisory_xact_lock(${LOCK_CLASS}, ${LOCK_OBJECT}) AS locked + ` + if (!lock?.locked) throw new Error('DATABASE_MIGRATION_BUSY') + const before = await status(transaction, catalog) + if (before.pending.length === 0) return before + await transaction`CREATE SCHEMA IF NOT EXISTS drizzle` + await transaction` + CREATE TABLE IF NOT EXISTS drizzle.__drizzle_migrations ( + id SERIAL PRIMARY KEY, + hash text NOT NULL, + created_at bigint + ) + ` + for (const migration of catalog.slice(before.applied.length)) { + // These are reviewed local migration artifacts, never request-supplied SQL. + for (const statement of migration.statements) await transaction.unsafe(statement) + await transaction` + INSERT INTO drizzle.__drizzle_migrations (hash, created_at) + VALUES (${migration.hash}, ${migration.when}) + ` + } + return status(transaction, catalog) +} + +/** Applies migrations inside a caller-owned transaction, allowing an atomic bootstrap. */ +export async function migrateDatabaseInTransaction( + transaction: TransactionSql, + migrationsFolder = DATABASE_MIGRATIONS_FOLDER, +): Promise { + return migrateCatalogInTransaction(transaction, readCatalog(migrationsFolder)) +} diff --git a/apps/indexer/src/lib/db/provision.ts b/apps/indexer/src/lib/db/provision.ts index 0fec561..3266cd6 100644 --- a/apps/indexer/src/lib/db/provision.ts +++ b/apps/indexer/src/lib/db/provision.ts @@ -1,25 +1,37 @@ -// Copied from packages/db/src/provision.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). -// Diverges by design (provisioning is grants-only: the managed PostgreSQL service owns the runtime users and their passwords, so this copy neither creates roles nor sets passwords); source sha256:ad624bb5a8d9403dda699767be589440680ac21bb4e2a3d7cfd648fb592b0fb5. +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. +// Provisioning is grants-only: DigitalOcean owns runtime users and passwords. +import type { TransactionSql } from 'postgres' + import type { DatabaseClient } from './client.js' export const XCS_INDEXER_DATABASE_ROLE = 'xcs_indexer' as const export const XCS_API_DATABASE_ROLE = 'xcs_api' as const +export const XCS_PAYLOAD_WRITER_DATABASE_ROLE = 'xcs_payload_writer' as const export const XCS_MONITOR_DATABASE_ROLE = 'xcs_monitor' as const +export const XCS_APP_DATABASE_ROLE = 'xcs_app' as const +export const XCS_ADMIN_APP_DATABASE_ROLE = 'xcs_admin_app' as const +export const XCS_NOTIFIER_DATABASE_ROLE = 'xcs_notifier' as const +export const XCS_ISSUER_DATABASE_ROLE = 'xcs_issuer' as const export const XCS_DATABASE_CLUSTER_SCOPE = 'dedicated' as const +const PROVISION_LOCK_CLASS_ID = 1_480_807_217 +const PROVISION_LOCK_OBJECT_ID = 1 + export const XCS_RUNTIME_DATABASE_ROLES = [ XCS_INDEXER_DATABASE_ROLE, XCS_API_DATABASE_ROLE, + XCS_PAYLOAD_WRITER_DATABASE_ROLE, XCS_MONITOR_DATABASE_ROLE, + XCS_APP_DATABASE_ROLE, + XCS_ADMIN_APP_DATABASE_ROLE, + XCS_NOTIFIER_DATABASE_ROLE, + XCS_ISSUER_DATABASE_ROLE, ] as const export const XCS_INDEXER_DATABASE_CONNECTION_LIMIT = 12 export const XCS_API_DATABASE_CONNECTION_LIMIT = 12 export const XCS_MONITOR_DATABASE_CONNECTION_LIMIT = 3 -const PROVISION_LOCK_CLASS_ID = 1_480_807_217 -const PROVISION_LOCK_OBJECT_ID = 1 - export interface RuntimeDatabaseProvisioning { clusterScope: typeof XCS_DATABASE_CLUSTER_SCOPE } @@ -125,6 +137,14 @@ const XCS_RUNTIME_DATABASE_ROLE_RESOURCE_CONTROLS: Readonly< idle_in_transaction_session_timeout: '30s', }, }, + xcs_payload_writer: { + connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, + settings: { + statement_timeout: '30s', + lock_timeout: '15s', + idle_in_transaction_session_timeout: '30s', + }, + }, xcs_monitor: { connectionLimit: XCS_MONITOR_DATABASE_CONNECTION_LIMIT, settings: { @@ -133,6 +153,38 @@ const XCS_RUNTIME_DATABASE_ROLE_RESOURCE_CONTROLS: Readonly< idle_in_transaction_session_timeout: '30s', }, }, + xcs_app: { + connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, + settings: { + statement_timeout: '30s', + lock_timeout: '15s', + idle_in_transaction_session_timeout: '30s', + }, + }, + xcs_admin_app: { + connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, + settings: { + statement_timeout: '30s', + lock_timeout: '15s', + idle_in_transaction_session_timeout: '30s', + }, + }, + xcs_notifier: { + connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, + settings: { + statement_timeout: '30s', + lock_timeout: '15s', + idle_in_transaction_session_timeout: '30s', + }, + }, + xcs_issuer: { + connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, + settings: { + statement_timeout: '30s', + lock_timeout: '15s', + idle_in_transaction_session_timeout: '30s', + }, + }, } /** A resource control the cluster refused and that is not already in effect. @@ -224,21 +276,85 @@ function unsafeRoleNames(findings: readonly string[]): string[] { } const REVOKE_CURRENT_DATABASE_ACCESS_SQL = ` - REVOKE ALL PRIVILEGES ON SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor; - REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor; - REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor; - REVOKE ALL PRIVILEGES ON ALL ROUTINES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor; + REVOKE ALL PRIVILEGES ON SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer; + REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer; + REVOKE ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer; + REVOKE ALL PRIVILEGES ON ALL ROUTINES IN SCHEMA public FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer; REVOKE CREATE ON SCHEMA public FROM PUBLIC; DO $xcs_database_grants$ BEGIN - EXECUTE format('REVOKE ALL PRIVILEGES ON DATABASE %I FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor', current_database()); - EXECUTE format('GRANT CONNECT ON DATABASE %I TO xcs_indexer, xcs_api, xcs_monitor', current_database()); + EXECUTE format('REVOKE ALL PRIVILEGES ON DATABASE %I FROM PUBLIC, xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer', current_database()); + EXECUTE format('GRANT CONNECT ON DATABASE %I TO xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer', current_database()); END $xcs_database_grants$; ` +// Table-level REVOKE does not remove column grants. Normalize every runtime role +// so a previous release cannot retain broader column-level privileges. +const REVOKE_RUNTIME_COLUMNS_SQL = ` + DO $xcs_app_columns$ + DECLARE relation record; + BEGIN + FOR relation IN + SELECT table_name, string_agg(format('%I', column_name), ', ' ORDER BY ordinal_position) AS columns + FROM information_schema.columns WHERE table_schema = 'public' GROUP BY table_name + LOOP + EXECUTE format('REVOKE SELECT (%2$s), INSERT (%2$s), UPDATE (%2$s), REFERENCES (%2$s) ON TABLE public.%1$I FROM xcs_indexer, xcs_api, xcs_monitor, xcs_payload_writer, xcs_app, xcs_admin_app, xcs_notifier, xcs_issuer', relation.table_name, relation.columns); + END LOOP; + END + $xcs_app_columns$; +` + +const GRANT_APPLICATION_ACCESS_SQL = ` + GRANT USAGE ON SCHEMA public TO xcs_app; + GRANT SELECT ON TABLE app_users, app_user_roles, app_wallets TO xcs_app; + GRANT SELECT (id, responsible_user_id, name, status) ON TABLE app_organizations TO xcs_app; + GRANT SELECT (organization_id, role, status) ON TABLE app_organization_applications TO xcs_app; + GRANT INSERT (identity_issuer, identity_subject, email, email_verified_at, display_name) ON TABLE app_users TO xcs_app; + GRANT UPDATE (email, email_verified_at, display_name) ON TABLE app_users TO xcs_app; + GRANT INSERT (user_id) ON TABLE app_user_roles TO xcs_app; + GRANT INSERT (user_id, network_id, address, verified_at) ON TABLE app_wallets TO xcs_app; + GRANT UPDATE (verified_at, revoked_at) ON TABLE app_wallets TO xcs_app; + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE app_sessions, app_auth_transactions, app_wallet_challenges TO xcs_app; +` + +// The portal administrator pool is separate from the bootstrap superuser xcs_admin. +const GRANT_ADMIN_ACCESS_SQL = ` + GRANT USAGE ON SCHEMA public TO xcs_admin_app; + GRANT SELECT (id, email, email_verified_at, display_name, status) ON app_users TO xcs_admin_app; + GRANT SELECT ON app_user_roles, app_sessions, app_organizations, app_organization_applications, app_documents, app_wallets TO xcs_admin_app; + GRANT SELECT, INSERT ON app_admin_decisions, app_admin_notifications TO xcs_admin_app; + GRANT UPDATE (status, reviewed_by, reviewed_at, review_reason, revision) ON app_organization_applications TO xcs_admin_app; + GRANT UPDATE (status, recipient_email, attempt_id, claimed_at, error_code) ON app_admin_notifications TO xcs_admin_app; +` +const GRANT_NOTIFIER_ACCESS_SQL = ` + GRANT USAGE ON SCHEMA public TO xcs_notifier; + GRANT SELECT ON app_admin_notifications, app_admin_decisions TO xcs_notifier; + GRANT SELECT (id, name) ON app_organizations TO xcs_notifier; + GRANT SELECT (id, email, email_verified_at, status) ON app_users TO xcs_notifier; + GRANT UPDATE (status, attempts, attempt_id, claimed_at, sent_at, error_code, recipient_email) ON app_admin_notifications TO xcs_notifier; +` + +const GRANT_ISSUER_ACCESS_SQL = ` + GRANT USAGE ON SCHEMA public TO xcs_issuer; + GRANT SELECT (id, email, email_verified_at, display_name, status) ON app_users TO xcs_issuer; + GRANT SELECT ON app_sessions, app_wallets, app_organizations, app_organization_applications, + app_schema_metadata, app_invites, app_credential_metadata, app_issuer_payloads, + app_invite_deliveries, app_presentations, network_profiles, schemas, schema_events, + credential_generations, credential_events, ledger_checkpoints, indexer_status, app_verifier_history, app_presentation_challenges, app_presentation_proofs TO xcs_issuer; + GRANT INSERT (id, responsible_user_id, name) ON app_organizations TO xcs_issuer; + GRANT INSERT (organization_id, role, website, contact, jurisdiction, description, purpose) ON app_organization_applications TO xcs_issuer; + GRANT INSERT (id, organization_id, application_role, storage_key, mime_type, byte_length, sha256, uploaded_by) ON app_documents TO xcs_issuer; + GRANT INSERT ON app_schema_metadata, app_invites, app_credential_metadata, app_issuer_payloads, app_invite_deliveries TO xcs_issuer; + GRANT UPDATE (token_hash, expires_at, revoked_at, claimed_by, claimed_at) ON app_invites TO xcs_issuer; + GRANT UPDATE (status, error_code) ON app_invite_deliveries TO xcs_issuer; + GRANT INSERT ON app_presentations, app_verifier_history, app_presentation_challenges, app_presentation_proofs TO xcs_issuer; + GRANT DELETE ON app_presentation_challenges TO xcs_issuer; + GRANT UPDATE (revoked_at) ON app_presentations TO xcs_issuer; +` + const GRANT_RUNTIME_ACCESS_SQL = ` - GRANT USAGE ON SCHEMA public TO xcs_indexer, xcs_api; + GRANT USAGE ON SCHEMA public TO xcs_indexer, xcs_api, xcs_payload_writer; GRANT SELECT, INSERT ON TABLE network_profiles, ledger_checkpoints, schema_events, schemas, credential_events @@ -257,7 +373,8 @@ const GRANT_RUNTIME_ACCESS_SQL = ` network_profiles, ledger_checkpoints, indexer_status, indexer_incidents, schema_events, schemas, credential_generations, credential_events TO xcs_api; - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pin_challenges, demo_pins TO xcs_api; + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pin_challenges, demo_pins TO xcs_payload_writer; + GRANT SELECT, INSERT ON TABLE hosted_payloads, hosted_payload_publications TO xcs_payload_writer; ` const RUNTIME_ROLE_ATTRIBUTES_SQL = ` @@ -272,7 +389,7 @@ const RUNTIME_ROLE_ATTRIBUTES_SQL = ` rolconnlimit AS "connectionLimit", rolconfig AS "configuration" FROM pg_roles - WHERE rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + WHERE rolname IN ('xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer') ORDER BY rolname ` @@ -281,8 +398,8 @@ const RUNTIME_ROLE_MEMBERSHIPS_SQL = ` FROM pg_auth_members auth_membership JOIN pg_roles granted_role ON granted_role.oid = auth_membership.roleid JOIN pg_roles member_role ON member_role.oid = auth_membership.member - WHERE granted_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') - OR member_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + WHERE granted_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer') + OR member_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer') ORDER BY granted_role.rolname, member_role.rolname ` @@ -292,135 +409,147 @@ function settingOf(configuration: readonly string[] | null, name: string): strin return entry === undefined ? undefined : entry.slice(prefix.length) } -export async function provisionRuntimeDatabasePrivileges( - client: DatabaseClient, +export async function prepareRuntimeDatabaseProvisioningInTransaction( + sql: TransactionSql, provisioning: RuntimeDatabaseProvisioning, -): Promise { +): Promise { parseDatabaseClusterScope(provisioning.clusterScope) + await sql`SELECT pg_advisory_xact_lock(${PROVISION_LOCK_CLASS_ID}, ${PROVISION_LOCK_OBJECT_ID})` - return await client.sql.begin(async (sql) => { - await sql`SELECT pg_advisory_xact_lock(${PROVISION_LOCK_CLASS_ID}, ${PROVISION_LOCK_OBJECT_ID})` - - // The administrator is read from the live session, not hard-coded: it is - // `doadmin` on DigitalOcean, something else on another provider and on a - // developer's own PostgreSQL. - const [session] = await sql<{ administrator: string }[]>` - SELECT current_user AS "administrator" - ` - const administrator = session?.administrator - if (administrator === undefined) { - throw new DatabaseBootstrapConfigurationError( - 'The database session reported no current user, so the cluster administrator could not ' + - 'be identified. Provisioning verifies role memberships against it and cannot continue.', - ) - } + const [session] = await sql<{ administrator: string }[]>` + SELECT current_user AS "administrator" + ` + const administrator = session?.administrator + if (administrator === undefined) { + throw new DatabaseBootstrapConfigurationError( + 'The database session reported no current user, so the cluster administrator could not ' + + 'be identified. Provisioning verifies role memberships against it and cannot continue.', + ) + } - const present = await sql.unsafe(RUNTIME_ROLE_ATTRIBUTES_SQL) - const found = new Set(present.map((row) => row.roleName)) - const missing = XCS_RUNTIME_DATABASE_ROLES.filter((role) => !found.has(role)) - if (missing.length > 0) { - throw new MissingRuntimeDatabaseRolesError(missing) - } + const present = await sql.unsafe(RUNTIME_ROLE_ATTRIBUTES_SQL) + const found = new Set(present.map((row) => row.roleName)) + const missing = XCS_RUNTIME_DATABASE_ROLES.filter((role) => !found.has(role)) + if (missing.length > 0) throw new MissingRuntimeDatabaseRolesError(missing) + + const memberships = await sql.unsafe(RUNTIME_ROLE_MEMBERSHIPS_SQL) + const findings = [ + ...unsafeRoleAttributeFindings(present), + ...unsafeRoleMembershipFindings(memberships, administrator), + ] + if (findings.length > 0) { + throw new UnsafeRuntimeDatabaseRolesError(findings, unsafeRoleNames(findings)) + } - const memberships = await sql.unsafe(RUNTIME_ROLE_MEMBERSHIPS_SQL) - const findings = [ - ...unsafeRoleAttributeFindings(present), - ...unsafeRoleMembershipFindings(memberships, administrator), - ] - if (findings.length > 0) { - throw new UnsafeRuntimeDatabaseRolesError(findings, unsafeRoleNames(findings)) - } + return administrator +} - await sql.unsafe(REVOKE_CURRENT_DATABASE_ACCESS_SQL) - await sql.unsafe(GRANT_RUNTIME_ACCESS_SQL) - - // pg_monitor is a predefined role the cluster owns. Passing it on requires - // the administrator to hold it WITH ADMIN OPTION, which a managed provider - // need not grant, so this is attempted rather than assumed: DigitalOcean - // refuses it at `check_role_membership_authorization`. A refusal costs the - // monitor role its server statistics, not the deployment, and it is named - // in the report rather than passing unnoticed. - const monitorGrants: { role: string; control: string; intended: string }[] = [] - try { - await sql.savepoint(async (scoped) => { - await scoped.unsafe('GRANT pg_monitor TO xcs_monitor WITH INHERIT TRUE, SET FALSE') - }) - } catch { - monitorGrants.push({ - role: XCS_MONITOR_DATABASE_ROLE, - control: 'pg_monitor membership', - intended: 'granted', - }) - } +export async function applyRuntimeDatabasePrivilegesInTransaction( + sql: TransactionSql, + administrator: string, +): Promise { + await sql.unsafe(REVOKE_CURRENT_DATABASE_ACCESS_SQL) + await sql.unsafe(REVOKE_RUNTIME_COLUMNS_SQL) + await sql.unsafe(GRANT_RUNTIME_ACCESS_SQL) + await sql.unsafe(GRANT_APPLICATION_ACCESS_SQL) + await sql.unsafe(GRANT_ADMIN_ACCESS_SQL) + await sql.unsafe(GRANT_NOTIFIER_ACCESS_SQL) + await sql.unsafe(GRANT_ISSUER_ACCESS_SQL) + + // pg_monitor is a predefined role the cluster owns. Passing it on requires + // the administrator to hold it WITH ADMIN OPTION, which a managed provider + // need not grant, so this is attempted rather than assumed: DigitalOcean + // refuses it at `check_role_membership_authorization`. A refusal costs the + // monitor role its server statistics, not the deployment, and it is named + // in the report rather than passing unnoticed. + const monitorGrants: { role: string; control: string; intended: string }[] = [] + try { + await sql.savepoint(async (scoped) => { + await scoped.unsafe('GRANT pg_monitor TO xcs_monitor WITH INHERIT TRUE, SET FALSE') + }) + } catch { + monitorGrants.push({ + role: XCS_MONITOR_DATABASE_ROLE, + control: 'pg_monitor membership', + intended: 'granted', + }) + } - // Resource controls last, one statement each inside its own savepoint, so - // a cluster that refuses one keeps the grants above. - const refused: { role: string; control: string; intended: string }[] = [] - for (const role of XCS_RUNTIME_DATABASE_ROLES) { - const controls = XCS_RUNTIME_DATABASE_ROLE_RESOURCE_CONTROLS[role] - const statements: { control: string; intended: string; statement: string }[] = [ - { - control: 'CONNECTION LIMIT', - intended: String(controls.connectionLimit), - statement: `ALTER ROLE ${role} WITH CONNECTION LIMIT ${controls.connectionLimit}`, - }, - ...Object.entries(controls.settings).map(([name, value]) => ({ - control: name, - intended: value, - statement: `ALTER ROLE ${role} SET ${name} = '${value}'`, - })), - ] - for (const { control, intended, statement } of statements) { - try { - await sql.savepoint(async (scoped) => { - await scoped.unsafe(statement) - }) - } catch { - refused.push({ role, control, intended }) - } + // Resource controls last, one statement each inside its own savepoint, so + // a cluster that refuses one keeps the grants above. + const refused: { role: string; control: string; intended: string }[] = [] + for (const role of XCS_RUNTIME_DATABASE_ROLES) { + const controls = XCS_RUNTIME_DATABASE_ROLE_RESOURCE_CONTROLS[role] + const statements: { control: string; intended: string; statement: string }[] = [ + { + control: 'CONNECTION LIMIT', + intended: String(controls.connectionLimit), + statement: `ALTER ROLE ${role} WITH CONNECTION LIMIT ${controls.connectionLimit}`, + }, + ...Object.entries(controls.settings).map(([name, value]) => ({ + control: name, + intended: value, + statement: `ALTER ROLE ${role} SET ${name} = '${value}'`, + })), + ] + for (const { control, intended, statement } of statements) { + try { + await sql.savepoint(async (scoped) => { + await scoped.unsafe(statement) + }) + } catch { + refused.push({ role, control, intended }) } } + } - // A refusal is only a loss if the intended value is not already in effect. - let unappliedResourceControls: UnappliedRuntimeRoleResourceControl[] = [] - if (refused.length > 0) { - const effective = await sql.unsafe(RUNTIME_ROLE_ATTRIBUTES_SQL) - const byRole = new Map(effective.map((row) => [row.roleName, row])) - unappliedResourceControls = refused.flatMap(({ role, control, intended }) => { - const row = byRole.get(role) - const actual = - control === 'CONNECTION LIMIT' - ? row === undefined - ? undefined - : String(row.connectionLimit) - : settingOf(row?.configuration ?? null, control) - return actual === intended ? [] : [{ role, control, intended, actual: actual ?? 'unset' }] - }) - } + // A refusal is only a loss if the intended value is not already in effect. + let unappliedResourceControls: UnappliedRuntimeRoleResourceControl[] = [] + if (refused.length > 0) { + const effective = await sql.unsafe(RUNTIME_ROLE_ATTRIBUTES_SQL) + const byRole = new Map(effective.map((row) => [row.roleName, row])) + unappliedResourceControls = refused.flatMap(({ role, control, intended }) => { + const row = byRole.get(role) + const actual = + control === 'CONNECTION LIMIT' + ? row === undefined + ? undefined + : String(row.connectionLimit) + : settingOf(row?.configuration ?? null, control) + return actual === intended ? [] : [{ role, control, intended, actual: actual ?? 'unset' }] + }) + } - // A refused pg_monitor grant is only a loss when the membership is absent. - const unappliedMonitorGrants: UnappliedRuntimeRoleResourceControl[] = [] - if (monitorGrants.length > 0) { - const [held] = await sql>` - SELECT EXISTS ( - SELECT 1 - FROM pg_auth_members membership - JOIN pg_roles granted_role ON granted_role.oid = membership.roleid - JOIN pg_roles member_role ON member_role.oid = membership.member - WHERE granted_role.rolname = 'pg_monitor' - AND member_role.rolname = ${XCS_MONITOR_DATABASE_ROLE} - ) AS present - ` - if (held?.present !== true) { - unappliedMonitorGrants.push( - ...monitorGrants.map((entry) => ({ ...entry, actual: 'absent' })), - ) - } + // A refused pg_monitor grant is only a loss when the membership is absent. + const unappliedMonitorGrants: UnappliedRuntimeRoleResourceControl[] = [] + if (monitorGrants.length > 0) { + const [held] = await sql>` + SELECT EXISTS ( + SELECT 1 + FROM pg_auth_members membership + JOIN pg_roles granted_role ON granted_role.oid = membership.roleid + JOIN pg_roles member_role ON member_role.oid = membership.member + WHERE granted_role.rolname = 'pg_monitor' + AND member_role.rolname = ${XCS_MONITOR_DATABASE_ROLE} + ) AS present + ` + if (held?.present !== true) { + unappliedMonitorGrants.push(...monitorGrants.map((entry) => ({ ...entry, actual: 'absent' }))) } + } - return { - administrator, - unappliedResourceControls: [...unappliedResourceControls, ...unappliedMonitorGrants], - } + return { + administrator, + unappliedResourceControls: [...unappliedResourceControls, ...unappliedMonitorGrants], + } +} +export async function provisionRuntimeDatabasePrivileges( + client: DatabaseClient, + provisioning: RuntimeDatabaseProvisioning, +): Promise { + parseDatabaseClusterScope(provisioning.clusterScope) + return await client.sql.begin(async (sql) => { + const administrator = await prepareRuntimeDatabaseProvisioningInTransaction(sql, provisioning) + return await applyRuntimeDatabasePrivilegesInTransaction(sql, administrator) }) } diff --git a/apps/indexer/src/lib/db/transactions.ts b/apps/indexer/src/lib/db/transactions.ts index 4164063..177eba7 100644 --- a/apps/indexer/src/lib/db/transactions.ts +++ b/apps/indexer/src/lib/db/transactions.ts @@ -1,4 +1,4 @@ -// Copied from packages/db/src/transactions.ts at 61fb809; keep in sync by hand (see CONTRIBUTING.md). +// Not a vendored copy (retired source): application-local database implementation maintained with db/schema. import type { XcsDatabase } from './client.js' const RETRYABLE_TRANSACTION_ERROR_CODES = new Set(['40001', '40P01']) diff --git a/apps/indexer/src/lib/xcs/claims.ts b/apps/indexer/src/lib/xcs/claims.ts index 75c3bf8..939b03f 100644 --- a/apps/indexer/src/lib/xcs/claims.ts +++ b/apps/indexer/src/lib/xcs/claims.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/claims.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/claims.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { base64urlnopad } from '@scure/base' import { isValidClassicAddress } from 'xrpl' @@ -33,46 +33,46 @@ function isCanonicalBase64Url(value: string): boolean { } function parseValue(value: unknown, descriptor: FieldDescriptor, path: string): JsonValue { - if (value === null) return fail('INVALID_CLAIMS', 'null is not allowed', path) + if (value === null) return fail('CLAIMS_INVALID', 'null is not allowed', path) switch (descriptor.type) { case 'string': - if (typeof value !== 'string') return fail('INVALID_CLAIMS', 'Expected a string', path) + if (typeof value !== 'string') return fail('CLAIMS_INVALID', 'Expected a string', path) return value case 'bool': - if (typeof value !== 'boolean') return fail('INVALID_CLAIMS', 'Expected a boolean', path) + if (typeof value !== 'boolean') return fail('CLAIMS_INVALID', 'Expected a boolean', path) return value case 'uint': { if (typeof value !== 'string' || !UNSIGNED_INTEGER.test(value)) { - return fail('INVALID_CLAIMS', 'Expected a canonical unsigned decimal string', path) + return fail('CLAIMS_INVALID', 'Expected a canonical unsigned decimal string', path) } if (value.length > 78 || BigInt(value) > UINT256_MAX) { - return fail('INVALID_CLAIMS', 'Unsigned integer exceeds 256 bits', path) + return fail('CLAIMS_INVALID', 'Unsigned integer exceeds 256 bits', path) } return value } case 'int': { if (typeof value !== 'string' || !SIGNED_INTEGER.test(value)) { - return fail('INVALID_CLAIMS', 'Expected a canonical signed decimal string', path) + return fail('CLAIMS_INVALID', 'Expected a canonical signed decimal string', path) } const integer = BigInt(value) if (value.length > 79 || integer < INT256_MIN || integer > INT256_MAX) { - return fail('INVALID_CLAIMS', 'Signed integer exceeds 256 bits', path) + return fail('CLAIMS_INVALID', 'Signed integer exceeds 256 bits', path) } return value } case 'bytes': if (typeof value !== 'string' || !isCanonicalBase64Url(value)) { - return fail('INVALID_CLAIMS', 'Expected unpadded canonical base64url', path) + return fail('CLAIMS_INVALID', 'Expected unpadded canonical base64url', path) } return value case 'address': if (typeof value !== 'string' || !isValidClassicAddress(value)) { - return fail('INVALID_CLAIMS', 'Expected an XRPL classic address', path) + return fail('CLAIMS_INVALID', 'Expected an XRPL classic address', path) } return value case 'array': - if (!Array.isArray(value)) return fail('INVALID_CLAIMS', 'Expected an array', path) + if (!Array.isArray(value)) return fail('CLAIMS_INVALID', 'Expected an array', path) return value.map((item, index) => parseValue(item, descriptor.items, `${path}[${index}]`)) case 'object': return parseObject(value, descriptor.fields, path) @@ -80,10 +80,10 @@ function parseValue(value: unknown, descriptor: FieldDescriptor, path: string): } function parseObject(input: unknown, fields: SchemaFields, path: string): JsonObject { - if (!isRecord(input)) return fail('INVALID_CLAIMS', 'Expected an object', path) + if (!isRecord(input)) return fail('CLAIMS_INVALID', 'Expected an object', path) for (const name of Object.keys(input)) { if (!Object.hasOwn(fields, name)) { - return fail('INVALID_CLAIMS', `Unknown claim ${name}`, `${path}.${name}`) + return fail('CLAIMS_INVALID', `Unknown claim ${name}`, `${path}.${name}`) } } @@ -91,7 +91,7 @@ function parseObject(input: unknown, fields: SchemaFields, path: string): JsonOb for (const [name, descriptor] of Object.entries(fields)) { if (!Object.hasOwn(input, name)) { if (descriptor.optional === true) continue - return fail('INVALID_CLAIMS', `Missing required claim ${name}`, `${path}.${name}`) + return fail('CLAIMS_INVALID', `Missing required claim ${name}`, `${path}.${name}`) } claims[name] = parseValue(input[name], descriptor, `${path}.${name}`) } @@ -101,3 +101,15 @@ function parseObject(input: unknown, fields: SchemaFields, path: string): JsonOb export function parseClaims(input: unknown, fields: SchemaFields): JsonObject { return parseObject(input, fields, '$.claims') } + +export function validateClaims( + input: unknown, + schema: SchemaFields | { fields: SchemaFields; definition?: unknown; lineage?: unknown }, +): JsonObject { + const resolved = schema as { fields?: unknown; definition?: unknown; lineage?: unknown } + const fields = + resolved.definition !== undefined && resolved.lineage !== undefined + ? (resolved.fields as SchemaFields) + : (schema as SchemaFields) + return parseClaims(input, fields) +} diff --git a/apps/indexer/src/lib/xcs/credential.ts b/apps/indexer/src/lib/xcs/credential.ts index 3c36374..043e758 100644 --- a/apps/indexer/src/lib/xcs/credential.ts +++ b/apps/indexer/src/lib/xcs/credential.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/credential.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/credential.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { isValidClassicAddress } from 'xrpl' import { parseClaims } from './claims.js' @@ -11,7 +11,9 @@ export interface CredentialContext { issuer: string subject: string schemaUid: string - fields: SchemaFields + fields?: SchemaFields + /** Backward-compatible XCS v0.1 context name. */ + schema?: SchemaFields } export interface CredentialPayload extends JsonObject { @@ -42,47 +44,52 @@ function isRecord(value: unknown): value is Record { function assertContext(context: CredentialContext): void { if (!isValidClassicAddress(context.issuer)) { - fail('INVALID_CREDENTIAL_PAYLOAD', 'Invalid issuer address', '$context.issuer') + fail('PAYLOAD_INVALID', 'Invalid issuer address', '$context.issuer') } if (!isValidClassicAddress(context.subject)) { - fail('INVALID_CREDENTIAL_PAYLOAD', 'Invalid subject address', '$context.subject') + fail('PAYLOAD_INVALID', 'Invalid subject address', '$context.subject') } if (!SCHEMA_UID.test(context.schemaUid)) { - fail('INVALID_CREDENTIAL_PAYLOAD', 'Invalid schema UID', '$context.schemaUid') + fail('PAYLOAD_INVALID', 'Invalid schema UID', '$context.schemaUid') } - if (!isRecord(context.fields)) { - fail('INVALID_CREDENTIAL_PAYLOAD', 'Invalid schema fields', '$context.fields') + if (!isRecord(context.fields ?? context.schema)) { + fail('PAYLOAD_INVALID', 'Invalid schema fields', '$context.fields') } } -function parsePayload(input: unknown, context: CredentialContext): CredentialPayload { +function contextFields(context: CredentialContext): SchemaFields { assertContext(context) + return (context.fields ?? context.schema) as SchemaFields +} + +function parsePayload(input: unknown, context: CredentialContext): CredentialPayload { + const fields = contextFields(context) if (!isRecord(input)) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Credential payload must be an object', '$') + return fail('PAYLOAD_INVALID', 'Credential payload must be an object', '$') } for (const property of Object.keys(input)) { if (!PAYLOAD_PROPERTIES.has(property)) { - return fail('INVALID_CREDENTIAL_PAYLOAD', `Unknown property ${property}`, `$.${property}`) + return fail('PAYLOAD_INVALID', `Unknown property ${property}`, `$.${property}`) } } if (input.xcsVersion !== '0.1') { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Unsupported XCS version', '$.xcsVersion') + return fail('PAYLOAD_INVALID', 'Unsupported XCS version', '$.xcsVersion') } if (input.issuer !== context.issuer) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Issuer does not match the credential', '$.issuer') + return fail('PAYLOAD_INVALID', 'Issuer does not match the credential', '$.issuer') } if (input.subject !== context.subject) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Subject does not match the credential', '$.subject') + return fail('PAYLOAD_INVALID', 'Subject does not match the credential', '$.subject') } if (input.schema !== context.schemaUid) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Schema does not match CredentialType', '$.schema') + return fail('PAYLOAD_INVALID', 'Schema does not match CredentialType', '$.schema') } return { xcsVersion: '0.1', issuer: context.issuer, subject: context.subject, schema: context.schemaUid, - claims: parseClaims(input.claims, context.fields), + claims: parseClaims(input.claims, fields), } } @@ -111,7 +118,21 @@ export function parseCredentialPayload( context: CredentialContext, ): CredentialPayload { const bytes = payloadBytes(content) - return parsePayload(parseCanonicalJson(bytes), context) + try { + return parsePayload(parseCanonicalJson(bytes), context) + } catch (error) { + if ( + error instanceof XcsError && + (error.code.startsWith('JSON_') || + error.code === 'UTF8_INVALID' || + error.code === 'CANONICALIZATION_UNSUPPORTED_VALUE') + ) { + return fail('PAYLOAD_INVALID', 'Credential payload must be canonical JSON', '$payload', { + cause: error.code, + }) + } + throw error + } } export function verifyCredentialPayload( diff --git a/apps/indexer/src/lib/xcs/errors.ts b/apps/indexer/src/lib/xcs/errors.ts index e7948bf..d9d59ef 100644 --- a/apps/indexer/src/lib/xcs/errors.ts +++ b/apps/indexer/src/lib/xcs/errors.ts @@ -1,16 +1,35 @@ -// Copied from packages/core/src/errors.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). -export type XcsErrorCode = - | 'INVALID_NETWORK_PROFILE' - | 'INVALID_SCHEMA' - | 'INVALID_SCHEMA_REFERENCE' - | 'INVALID_CLAIMS' - | 'INVALID_UID_INPUT' - | 'INVALID_CREDENTIAL_PAYLOAD' - | 'INVALID_PAYLOAD_URI' - | 'INVALID_RIPPLE_TIME' - | 'INVALID_JSON' - | 'NON_CANONICAL_JSON' - | 'UNSUPPORTED_JSON_VALUE' +// Copied from packages/core/src/errors.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). +export const XCS_ERROR_CODES = [ + 'JSON_INVALID', + 'JSON_DUPLICATE_KEY', + 'JSON_INVALID_UNICODE', + 'JSON_NON_IJSON_NUMBER', + 'CANONICALIZATION_UNSUPPORTED_VALUE', + 'HEX_INVALID', + 'UTF8_INVALID', + 'NETWORK_PROFILE_INVALID', + 'SCHEMA_INVALID', + 'SCHEMA_PARENT_NOT_FOUND', + 'SCHEMA_PARENT_NOT_PRIOR', + 'SCHEMA_PARENT_NETWORK_MISMATCH', + 'SCHEMA_OVERRIDE_FORBIDDEN', + 'SCHEMA_INHERITANCE_CYCLE', + 'SCHEMA_DEPTH_EXCEEDED', + 'SCHEMA_FIELD_LIMIT_EXCEEDED', + 'SCHEMA_SUPERSEDES_NOT_FOUND', + 'SCHEMA_SUPERSEDES_NOT_PRIOR', + 'SCHEMA_SUPERSEDES_PUBLISHER_MISMATCH', + 'SCHEMA_CATALOG_INVALID', + 'SCHEMA_CATALOG_LIMIT_EXCEEDED', + 'VERIFICATION_REPORT_INVALID', + 'UID_INPUT_INVALID', + 'CLAIMS_INVALID', + 'PAYLOAD_INVALID', + 'PAYLOAD_URI_INVALID', + 'RIPPLE_TIME_INVALID', +] as const + +export type XcsErrorCode = (typeof XCS_ERROR_CODES)[number] export class XcsError extends Error { readonly code: XcsErrorCode diff --git a/apps/indexer/src/lib/xcs/index.ts b/apps/indexer/src/lib/xcs/index.ts index cd9dfc0..7be864c 100644 --- a/apps/indexer/src/lib/xcs/index.ts +++ b/apps/indexer/src/lib/xcs/index.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/index.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/index.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). export { encodeCredentialPayload, parseCredentialPayload, @@ -9,27 +9,41 @@ export { type EncodedCredentialPayload, type PayloadRetrieval, } from './credential.js' -export { XcsError, type XcsErrorCode } from './errors.js' +export { XCS_ERROR_CODES, XcsError, type XcsErrorCode } from './errors.js' +export { parseClaims, validateClaims } from './claims.js' export { + canonicalize, canonicalJson, + decodeUtf8Hex, decodeHexUtf8, decodeUtf8, encodeCanonicalJson, encodeHexUtf8, encodeUtf8, + encodeUtf8Hex, parseCanonicalJson, parseJson, + parseJsonStrict, sha256Hex, utf8ByteLength, type JsonObject, type JsonPrimitive, type JsonValue, } from './json.js' -export { parseNetworkProfile, rippleTimeToIso, type NetworkProfile } from './network.js' export { + parseNetworkProfile, + rippleTimeToIso, + validateNetworkProfile, + type NetworkProfile, +} from './network.js' +export { + computePayloadSha256Hex, createHttpsPayloadUri, + createIpfsRawPayloadUri, createIpfsPayloadUri, + inspectPayloadUri, parsePayloadUri, + payloadDigest, verifyPayloadIntegrity, type HttpsPayloadUri, type IpfsPayloadUri, @@ -41,11 +55,19 @@ export { type CredentialLifecycleInput, type CredentialLifecycleState, } from './lifecycle.js' +export { + iso8601ToRippleTime, + RIPPLE_EPOCH_UNIX_SECONDS, + rippleTimeToIso8601, + rippleTimeToUnixSeconds, + unixSecondsToRippleTime, +} from './ripple-time.js' export { resolveSchema, type SchemaResolutionContext } from './schema-resolution.js' export { encodeSchema, parseSchema, parseSchemaBytes, + validateSchema, type ArrayFieldDescriptor, type FieldDescriptor, type ObjectFieldDescriptor, diff --git a/apps/indexer/src/lib/xcs/json.ts b/apps/indexer/src/lib/xcs/json.ts index c8133d3..53beded 100644 --- a/apps/indexer/src/lib/xcs/json.ts +++ b/apps/indexer/src/lib/xcs/json.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/json.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/json.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { sha256 } from '@noble/hashes/sha2.js' import { bytesToHex, hexToBytes } from '@noble/hashes/utils.js' import canonicalizeRfc8785 from 'canonicalize' @@ -22,12 +22,12 @@ function assertWellFormed(value: string, path: string): void { const codeUnit = value.charCodeAt(index) if (codeUnit >= 0xd800 && codeUnit <= 0xdbff) { const next = value.charCodeAt(index + 1) - if (next < 0xdc00 || next > 0xdfff) { - fail('UNSUPPORTED_JSON_VALUE', 'JSON strings must contain valid Unicode', path) + if (!(next >= 0xdc00 && next <= 0xdfff)) { + fail('JSON_INVALID_UNICODE', 'JSON strings must contain valid Unicode', path) } index += 1 } else if (codeUnit >= 0xdc00 && codeUnit <= 0xdfff) { - fail('UNSUPPORTED_JSON_VALUE', 'JSON strings must contain valid Unicode', path) + fail('JSON_INVALID_UNICODE', 'JSON strings must contain valid Unicode', path) } } } @@ -40,21 +40,25 @@ function assertJsonValueInternal(value: unknown, path: string, stack: Set)) { assertWellFormed(key, path) @@ -82,7 +86,11 @@ export function canonicalJson(value: unknown): string { assertJsonValue(value) const serialized = canonicalizeRfc8785(value) if (serialized === undefined) { - return fail('UNSUPPORTED_JSON_VALUE', 'Value cannot be represented as canonical JSON', '$') + return fail( + 'CANONICALIZATION_UNSUPPORTED_VALUE', + 'Value cannot be represented as canonical JSON', + '$', + ) } return serialized } @@ -99,7 +107,7 @@ export function decodeUtf8(bytes: Uint8Array): string { try { return decoder.decode(bytes) } catch (cause) { - return fail('INVALID_JSON', 'Input is not valid UTF-8', '$', { cause: String(cause) }) + return fail('UTF8_INVALID', 'Input is not valid UTF-8', '$', { cause: String(cause) }) } } @@ -124,7 +132,7 @@ export function parseJson(input: string | Uint8Array): JsonValue { allowEmptyContent: false, }) if (!root || errors.length > 0) { - return fail('INVALID_JSON', 'Input is not valid JSON', '$') + return fail('JSON_INVALID', 'Input is not valid JSON', '$') } rejectDuplicateObjectKeys(root) @@ -137,7 +145,7 @@ export function parseCanonicalJson(input: string | Uint8Array): JsonValue { const text = typeof input === 'string' ? input : decodeUtf8(input) const parsed = parseJson(text) if (canonicalJson(parsed) !== text) { - return fail('NON_CANONICAL_JSON', 'Input must use RFC 8785 canonical JSON', '$') + return fail('JSON_INVALID', 'Input must use RFC 8785 canonical JSON', '$') } assertJsonValue(parsed) return parsed @@ -150,10 +158,10 @@ function rejectDuplicateObjectKeys(node: Node): void { const [keyNode, valueNode] = property.children ?? [] const key = keyNode?.value if (typeof key !== 'string' || !valueNode) { - return fail('INVALID_JSON', 'Input contains an invalid JSON object', '$') + return fail('JSON_INVALID', 'Input contains an invalid JSON object', '$') } if (keys.has(key)) { - return fail('INVALID_JSON', `Input contains duplicate key ${key}`, '$') + return fail('JSON_DUPLICATE_KEY', `Input contains duplicate key ${key}`, '$') } keys.add(key) rejectDuplicateObjectKeys(valueNode) @@ -167,3 +175,9 @@ export function utf8ByteLength(value: string): number { assertWellFormed(value, '$') return encoder.encode(value).length } + +/** Backward-compatible names from the normative XCS v0.1 interface. */ +export const canonicalize = canonicalJson +export const parseJsonStrict = parseJson +export const encodeUtf8Hex = (value: string): string => encodeHexUtf8(value).toUpperCase() +export const decodeUtf8Hex = decodeHexUtf8 diff --git a/apps/indexer/src/lib/xcs/lifecycle.ts b/apps/indexer/src/lib/xcs/lifecycle.ts index d0cde5e..4b52cb6 100644 --- a/apps/indexer/src/lib/xcs/lifecycle.ts +++ b/apps/indexer/src/lib/xcs/lifecycle.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/lifecycle.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/lifecycle.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { fail } from './errors.js' export type CredentialLifecycleState = 'pending' | 'active' | 'expired' | 'deleted' @@ -14,7 +14,7 @@ const UINT32_MAX = 0xffff_ffff function requireRippleTime(value: number): void { if (!Number.isInteger(value) || value < 0 || value > UINT32_MAX) { - return fail('INVALID_RIPPLE_TIME', 'Credential lifecycle time must be a uint32', '$time') + return fail('RIPPLE_TIME_INVALID', 'Credential lifecycle time must be a uint32', '$time') } } diff --git a/apps/indexer/src/lib/xcs/network.ts b/apps/indexer/src/lib/xcs/network.ts index 876c16e..07c72f1 100644 --- a/apps/indexer/src/lib/xcs/network.ts +++ b/apps/indexer/src/lib/xcs/network.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/network.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/network.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { isValidClassicAddress, rippleTimeToISOTime as xrplRippleTimeToIso } from 'xrpl' import { fail } from './errors.js' @@ -35,48 +35,48 @@ function isUint32(value: unknown): value is number { export function parseNetworkProfile(input: unknown): NetworkProfile { if (!isRecord(input)) { - return fail('INVALID_NETWORK_PROFILE', 'Network profile must be an object', '$') + return fail('NETWORK_PROFILE_INVALID', 'Network profile must be an object', '$') } for (const key of Object.keys(input)) { if (!PROFILE_PROPERTIES.has(key)) { - return fail('INVALID_NETWORK_PROFILE', `Unknown property ${key}`, `$.${key}`) + return fail('NETWORK_PROFILE_INVALID', `Unknown property ${key}`, `$.${key}`) } } if ( typeof input.profileId !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,127}$/.test(input.profileId) ) { - return fail('INVALID_NETWORK_PROFILE', 'Invalid profileId', '$.profileId') + return fail('NETWORK_PROFILE_INVALID', 'Invalid profileId', '$.profileId') } if (input.xcsVersion !== '0.1') { - return fail('INVALID_NETWORK_PROFILE', 'Unsupported XCS version', '$.xcsVersion') + return fail('NETWORK_PROFILE_INVALID', 'Unsupported XCS version', '$.xcsVersion') } if (!isUint32(input.networkId)) { - return fail('INVALID_NETWORK_PROFILE', 'networkId must be a uint32', '$.networkId') + return fail('NETWORK_PROFILE_INVALID', 'networkId must be a uint32', '$.networkId') } if ( typeof input.requiredAmendment !== 'string' || !/^[0-9a-fA-F]{64}$/.test(input.requiredAmendment) ) { return fail( - 'INVALID_NETWORK_PROFILE', + 'NETWORK_PROFILE_INVALID', 'requiredAmendment must be a 32-byte hexadecimal value', '$.requiredAmendment', ) } if (typeof input.registryAddress !== 'string' || !isValidClassicAddress(input.registryAddress)) { - return fail('INVALID_NETWORK_PROFILE', 'Invalid XRPL registry address', '$.registryAddress') + return fail('NETWORK_PROFILE_INVALID', 'Invalid XRPL registry address', '$.registryAddress') } if (input.registrationAmountDrops !== '1') { return fail( - 'INVALID_NETWORK_PROFILE', + 'NETWORK_PROFILE_INVALID', 'registrationAmountDrops must equal "1"', '$.registrationAmountDrops', ) } if (!isUint32(input.activationLedgerIndex) || input.activationLedgerIndex === 0) { return fail( - 'INVALID_NETWORK_PROFILE', + 'NETWORK_PROFILE_INVALID', 'activationLedgerIndex must be a positive uint32', '$.activationLedgerIndex', ) @@ -86,7 +86,7 @@ export function parseNetworkProfile(input: unknown): NetworkProfile { !/^[0-9a-fA-F]{64}$/.test(input.activationLedgerHash) ) { return fail( - 'INVALID_NETWORK_PROFILE', + 'NETWORK_PROFILE_INVALID', 'activationLedgerHash must be a 32-byte hexadecimal value', '$.activationLedgerHash', ) @@ -104,9 +104,12 @@ export function parseNetworkProfile(input: unknown): NetworkProfile { } } +/** Normative XCS v0.1 name retained for existing consumers. */ +export const validateNetworkProfile = parseNetworkProfile + export function rippleTimeToIso(rippleTime: number): string { if (!isUint32(rippleTime)) { - return fail('INVALID_RIPPLE_TIME', 'Ripple time must be a uint32', '$time') + return fail('RIPPLE_TIME_INVALID', 'Ripple time must be a uint32', '$time') } return xrplRippleTimeToIso(rippleTime) } diff --git a/apps/indexer/src/lib/xcs/payload-uri.ts b/apps/indexer/src/lib/xcs/payload-uri.ts index a9e4a5c..7ce8f98 100644 --- a/apps/indexer/src/lib/xcs/payload-uri.ts +++ b/apps/indexer/src/lib/xcs/payload-uri.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/payload-uri.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/payload-uri.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { sha256 } from '@noble/hashes/sha2.js' import { bytesToHex } from '@noble/hashes/utils.js' import { CID } from 'multiformats/cid' @@ -30,7 +30,8 @@ export type PayloadUri = IpfsPayloadUri | HttpsPayloadUri export interface PayloadIntegrityResult { valid: boolean - expectedDigestHex: string + status: 'valid' | 'tampered' | 'invalid_uri' + expectedDigestHex?: string actualDigestHex: string } @@ -48,10 +49,10 @@ export function payloadBytes(content: string | Uint8Array): Uint8Array { if (typeof content === 'string') utf8ByteLength(content) const bytes = typeof content === 'string' ? new TextEncoder().encode(content) : content if (!(bytes instanceof Uint8Array)) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Payload must be UTF-8 text or bytes', '$payload') + return fail('PAYLOAD_INVALID', 'Payload must be UTF-8 text or bytes', '$payload') } if (bytes.length > MAX_PAYLOAD_BYTES) { - return fail('INVALID_CREDENTIAL_PAYLOAD', 'Payload exceeds 1 MiB', '$payload', { + return fail('PAYLOAD_INVALID', 'Payload exceeds 1 MiB', '$payload', { size: bytes.length, }) } @@ -185,7 +186,7 @@ function parseIpfsUri(uri: string): IpfsPayloadUri | undefined { cid.multihash.size !== 32 || cid.toString(base32.encoder) !== cidText ) { - return fail('INVALID_PAYLOAD_URI', 'Expected a canonical raw SHA-256 CIDv1', '$uri') + return fail('PAYLOAD_URI_INVALID', 'Expected a canonical raw SHA-256 CIDv1', '$uri') } return { kind: 'ipfs', @@ -195,30 +196,30 @@ function parseIpfsUri(uri: string): IpfsPayloadUri | undefined { } } catch (cause) { if (cause instanceof XcsError) throw cause - return fail('INVALID_PAYLOAD_URI', 'Invalid IPFS CID', '$uri', { cause: String(cause) }) + return fail('PAYLOAD_URI_INVALID', 'Invalid IPFS CID', '$uri', { cause: String(cause) }) } } export function parsePayloadUri(uri: string): PayloadUri { if (typeof uri !== 'string') { - return fail('INVALID_PAYLOAD_URI', 'URI must be a string', '$uri') + return fail('PAYLOAD_URI_INVALID', 'URI must be a string', '$uri') } const length = utf8ByteLength(uri) if (length < 1 || length > 256) { - return fail('INVALID_PAYLOAD_URI', 'URI must contain 1 to 256 UTF-8 bytes', '$uri') + return fail('PAYLOAD_URI_INVALID', 'URI must contain 1 to 256 UTF-8 bytes', '$uri') } const ipfs = parseIpfsUri(uri) if (ipfs !== undefined) return ipfs if (!uri.startsWith(HTTPS_PREFIX) || INVALID_RAW_HTTPS_CHARACTER.test(uri)) { - return fail('INVALID_PAYLOAD_URI', 'Expected a canonical HTTPS or IPFS URI', '$uri') + return fail('PAYLOAD_URI_INVALID', 'Expected a canonical HTTPS or IPFS URI', '$uri') } const remainder = uri.slice(HTTPS_PREFIX.length) const authorityEnd = remainder.search(/[/?#]/) const authority = authorityEnd === -1 ? remainder : remainder.slice(0, authorityEnd) if (authority === '' || authority.includes('@')) { return fail( - 'INVALID_PAYLOAD_URI', + 'PAYLOAD_URI_INVALID', 'HTTPS authority cannot be empty or contain userinfo', '$uri', ) @@ -226,14 +227,14 @@ export function parsePayloadUri(uri: string): PayloadUri { const normalizedUri = normalizeHttpsAuthority(uri) if (normalizedUri === undefined) { - return fail('INVALID_PAYLOAD_URI', 'Invalid HTTPS authority', '$uri') + return fail('PAYLOAD_URI_INVALID', 'Invalid HTTPS authority', '$uri') } const fetchUrl = buildFetchUrl(normalizedUri) let parsed: URL try { parsed = new URL(normalizedUri) } catch (cause) { - return fail('INVALID_PAYLOAD_URI', 'Invalid HTTPS URI', '$uri', { cause: String(cause) }) + return fail('PAYLOAD_URI_INVALID', 'Invalid HTTPS URI', '$uri', { cause: String(cause) }) } if ( fetchUrl === undefined || @@ -243,7 +244,7 @@ export function parsePayloadUri(uri: string): PayloadUri { !/^#xcs-sha256=[0-9a-f]{64}$/.test(parsed.hash) ) { return fail( - 'INVALID_PAYLOAD_URI', + 'PAYLOAD_URI_INVALID', 'HTTPS URI must end in #xcs-sha256=', '$uri', ) @@ -261,12 +262,12 @@ export function createHttpsPayloadUri(baseUrl: string, content: string | Uint8Ar try { url = new URL(baseUrl) } catch (cause) { - return fail('INVALID_PAYLOAD_URI', 'Invalid HTTPS base URL', '$url', { + return fail('PAYLOAD_URI_INVALID', 'Invalid HTTPS base URL', '$url', { cause: String(cause), }) } if (url.protocol !== 'https:' || url.username !== '' || url.password !== '' || url.hash !== '') { - return fail('INVALID_PAYLOAD_URI', 'Base URL must be fragment-free HTTPS', '$url') + return fail('PAYLOAD_URI_INVALID', 'Base URL must be fragment-free HTTPS', '$url') } url.hash = `xcs-sha256=${payloadDigest(content)}` const uri = url.toString() @@ -283,11 +284,25 @@ export function verifyPayloadIntegrity( content: string | Uint8Array, uri: string, ): PayloadIntegrityResult { - const expectedDigestHex = parsePayloadUri(uri).digestHex const actualDigestHex = payloadDigest(content) + let expectedDigestHex: string + try { + expectedDigestHex = parsePayloadUri(uri).digestHex + } catch (error) { + if (error instanceof XcsError && error.code === 'PAYLOAD_URI_INVALID') { + return { valid: false, status: 'invalid_uri', actualDigestHex } + } + throw error + } + const valid = expectedDigestHex === actualDigestHex return { - valid: expectedDigestHex === actualDigestHex, + valid, + status: valid ? 'valid' : 'tampered', expectedDigestHex, actualDigestHex, } } + +export const computePayloadSha256Hex = payloadDigest +export const createIpfsRawPayloadUri = createIpfsPayloadUri +export const inspectPayloadUri = parsePayloadUri diff --git a/apps/indexer/src/lib/xcs/ripple-time.ts b/apps/indexer/src/lib/xcs/ripple-time.ts new file mode 100644 index 0000000..9ab4448 --- /dev/null +++ b/apps/indexer/src/lib/xcs/ripple-time.ts @@ -0,0 +1,50 @@ +// Copied from packages/core/src/ripple-time.ts at 254c6a0; keep in sync by hand (see CONTRIBUTING.md). +import { fail } from './errors.js' + +export const RIPPLE_EPOCH_UNIX_SECONDS = 946_684_800 +const UINT32_MAX = 0xffff_ffff + +export function unixSecondsToRippleTime(unixSeconds: number): number { + if (!Number.isSafeInteger(unixSeconds)) { + return fail('RIPPLE_TIME_INVALID', 'Unix time must be an integer number of seconds', '$time') + } + const rippleTime = unixSeconds - RIPPLE_EPOCH_UNIX_SECONDS + if (rippleTime < 0 || rippleTime > UINT32_MAX) { + return fail('RIPPLE_TIME_INVALID', 'Unix time is outside the XRPL uint32 time range', '$time') + } + return rippleTime +} + +export function rippleTimeToUnixSeconds(rippleTime: number): number { + if (!Number.isInteger(rippleTime) || rippleTime < 0 || rippleTime > UINT32_MAX) { + return fail('RIPPLE_TIME_INVALID', 'Ripple time must be a uint32', '$time') + } + return rippleTime + RIPPLE_EPOCH_UNIX_SECONDS +} + +export function iso8601ToRippleTime(value: string): number { + if ( + typeof value !== 'string' || + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.000)?Z$/.test(value) + ) { + return fail( + 'RIPPLE_TIME_INVALID', + 'Expiration must be a UTC ISO-8601 timestamp with whole-second precision', + '$time', + ) + } + const milliseconds = Date.parse(value) + if (!Number.isFinite(milliseconds)) { + return fail('RIPPLE_TIME_INVALID', 'Expiration is not a real calendar date', '$time') + } + const normalized = new Date(milliseconds).toISOString() + const expected = value.includes('.') ? value : value.replace('Z', '.000Z') + if (normalized !== expected) { + return fail('RIPPLE_TIME_INVALID', 'Expiration is not a canonical calendar date', '$time') + } + return unixSecondsToRippleTime(milliseconds / 1000) +} + +export function rippleTimeToIso8601(rippleTime: number): string { + return new Date(rippleTimeToUnixSeconds(rippleTime) * 1000).toISOString() +} diff --git a/apps/indexer/src/lib/xcs/schema-resolution.ts b/apps/indexer/src/lib/xcs/schema-resolution.ts index c1b2c26..d0e612d 100644 --- a/apps/indexer/src/lib/xcs/schema-resolution.ts +++ b/apps/indexer/src/lib/xcs/schema-resolution.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/schema-resolution.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/schema-resolution.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { isValidClassicAddress } from 'xrpl' import { fail } from './errors.js' @@ -43,20 +43,23 @@ function requireReference( ): RegisteredSchema { const referenced = context.getSchema(uid) if (referenced === undefined || referenced.uid !== uid) { - return fail('INVALID_SCHEMA_REFERENCE', `${relation} schema was not found`, `$.${relation}`, { - uid, - }) + return fail( + relation === 'extends' ? 'SCHEMA_PARENT_NOT_FOUND' : 'SCHEMA_SUPERSEDES_NOT_FOUND', + `${relation} schema was not found`, + `$.${relation}`, + { uid }, + ) } if (referenced.networkId !== context.networkId) { return fail( - 'INVALID_SCHEMA_REFERENCE', + relation === 'extends' ? 'SCHEMA_PARENT_NETWORK_MISMATCH' : 'SCHEMA_SUPERSEDES_NOT_FOUND', `${relation} schema belongs to another network`, `$.${relation}`, ) } if (!isEarlier(referenced, context)) { return fail( - 'INVALID_SCHEMA_REFERENCE', + relation === 'extends' ? 'SCHEMA_PARENT_NOT_PRIOR' : 'SCHEMA_SUPERSEDES_NOT_PRIOR', `${relation} schema must be registered earlier`, `$.${relation}`, ) @@ -73,12 +76,11 @@ function resolve( let lineage: string[] = [] if (schema.extends !== undefined) { - if (visiting.size >= MAX_SCHEMA_DEPTH - 1 || visiting.has(schema.extends)) { - return fail( - 'INVALID_SCHEMA_REFERENCE', - 'Schema inheritance is cyclic or too deep', - '$.extends', - ) + if (visiting.has(schema.extends)) { + return fail('SCHEMA_INHERITANCE_CYCLE', 'Schema inheritance is cyclic', '$.extends') + } + if (visiting.size >= MAX_SCHEMA_DEPTH - 1) { + return fail('SCHEMA_DEPTH_EXCEEDED', 'Schema inheritance is too deep', '$.extends') } const parent = requireReference(schema.extends, 'extends', context) visiting.add(schema.extends) @@ -101,7 +103,7 @@ function resolve( for (const [name, descriptor] of Object.entries(schema.fields)) { if (Object.hasOwn(fields, name)) { return fail( - 'INVALID_SCHEMA_REFERENCE', + 'SCHEMA_OVERRIDE_FORBIDDEN', `Inherited field ${name} cannot be redefined`, `$.fields.${name}`, ) @@ -110,14 +112,18 @@ function resolve( } if (countSchemaFields(fields) > MAX_SCHEMA_FIELDS) { - return fail('INVALID_SCHEMA', `Resolved schema exceeds ${MAX_SCHEMA_FIELDS} fields`, '$.fields') + return fail( + 'SCHEMA_FIELD_LIMIT_EXCEEDED', + `Resolved schema exceeds ${MAX_SCHEMA_FIELDS} fields`, + '$.fields', + ) } if (schema.supersedes !== undefined) { const previous = requireReference(schema.supersedes, 'supersedes', context) if (previous.publisher !== context.publisher) { return fail( - 'INVALID_SCHEMA_REFERENCE', + 'SCHEMA_SUPERSEDES_PUBLISHER_MISMATCH', 'Only the original publisher may supersede a schema', '$.supersedes', ) @@ -138,7 +144,7 @@ export function resolveSchema( !isValidClassicAddress(context.publisher) || typeof context.getSchema !== 'function' ) { - return fail('INVALID_SCHEMA_REFERENCE', 'Invalid schema resolution context', '$context') + return fail('SCHEMA_INVALID', 'Invalid schema resolution context', '$context') } return resolve(parseSchema(input), context, new Set()) } diff --git a/apps/indexer/src/lib/xcs/schema-uid.ts b/apps/indexer/src/lib/xcs/schema-uid.ts index 2904155..886bbf3 100644 --- a/apps/indexer/src/lib/xcs/schema-uid.ts +++ b/apps/indexer/src/lib/xcs/schema-uid.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/schema-uid.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/schema-uid.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { sha256 } from '@noble/hashes/sha2.js' import { bytesToHex } from '@noble/hashes/utils.js' import { isValidClassicAddress } from 'xrpl' @@ -26,17 +26,17 @@ export function computeSchemaUid(input: SchemaUidInput): string { !isUint32(input.ledgerIndex) || !isUint32(input.transactionIndex) ) { - return fail('INVALID_UID_INPUT', 'Network and ledger coordinates must be uint32 values', '$') + return fail('UID_INPUT_INVALID', 'Network and ledger coordinates must be uint32 values', '$') } if (!/^[0-9a-fA-F]{64}$/.test(input.ledgerHash)) { return fail( - 'INVALID_UID_INPUT', + 'UID_INPUT_INVALID', 'ledgerHash must be a 32-byte hexadecimal hash', '$.ledgerHash', ) } if (!isValidClassicAddress(input.publisher)) { - return fail('INVALID_UID_INPUT', 'publisher must be an XRPL classic address', '$.publisher') + return fail('UID_INPUT_INVALID', 'publisher must be an XRPL classic address', '$.publisher') } const preimage = { diff --git a/apps/indexer/src/lib/xcs/schema.ts b/apps/indexer/src/lib/xcs/schema.ts index de394e4..87b38a2 100644 --- a/apps/indexer/src/lib/xcs/schema.ts +++ b/apps/indexer/src/lib/xcs/schema.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/schema.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/schema.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { fail } from './errors.js' import { encodeCanonicalJson, parseCanonicalJson, utf8ByteLength } from './json.js' @@ -76,17 +76,17 @@ function assertOnlyProperties( path: string, ): void { for (const key of Object.keys(value)) { - if (!allowed.has(key)) fail('INVALID_SCHEMA', `Unknown property ${key}`, `${path}.${key}`) + if (!allowed.has(key)) fail('SCHEMA_INVALID', `Unknown property ${key}`, `${path}.${key}`) } } function parseText(value: unknown, path: string, maxBytes: number): string { if (typeof value !== 'string' || CONTROL_CHARACTER.test(value)) { - return fail('INVALID_SCHEMA', `Expected 1 to ${maxBytes} UTF-8 bytes`, path) + return fail('SCHEMA_INVALID', `Expected 1 to ${maxBytes} UTF-8 bytes`, path) } const length = utf8ByteLength(value) if (length < 1 || length > maxBytes) { - return fail('INVALID_SCHEMA', `Expected 1 to ${maxBytes} UTF-8 bytes`, path) + return fail('SCHEMA_INVALID', `Expected 1 to ${maxBytes} UTF-8 bytes`, path) } return value } @@ -98,10 +98,10 @@ function parseDescriptor( count: { value: number }, ): FieldDescriptor { if (!isRecord(input) || typeof input.type !== 'string') { - return fail('INVALID_SCHEMA', 'Field descriptor must contain a type', path) + return fail('SCHEMA_INVALID', 'Field descriptor must contain a type', path) } if (input.optional !== undefined && typeof input.optional !== 'boolean') { - return fail('INVALID_SCHEMA', 'optional must be a boolean', `${path}.optional`) + return fail('SCHEMA_INVALID', 'optional must be a boolean', `${path}.optional`) } const optional = input.optional === true ? { optional: true as const } : {} @@ -112,7 +112,7 @@ function parseDescriptor( if (input.type === 'array') { assertOnlyProperties(input, new Set(['type', 'optional', 'items']), path) if (depth >= MAX_SCHEMA_DEPTH || input.items === undefined) { - return fail('INVALID_SCHEMA', 'Array descriptor is missing items or exceeds depth', path) + return fail('SCHEMA_INVALID', 'Array descriptor is missing items or exceeds depth', path) } count.value += 1 assertFieldLimit(count.value, `${path}.items`) @@ -125,7 +125,7 @@ function parseDescriptor( if (input.type === 'object') { assertOnlyProperties(input, new Set(['type', 'optional', 'fields']), path) if (depth >= MAX_SCHEMA_DEPTH) { - return fail('INVALID_SCHEMA', `Schema nesting exceeds ${MAX_SCHEMA_DEPTH}`, path) + return fail('SCHEMA_INVALID', `Schema nesting exceeds ${MAX_SCHEMA_DEPTH}`, path) } return { type: 'object', @@ -133,12 +133,16 @@ function parseDescriptor( ...optional, } } - return fail('INVALID_SCHEMA', `Unsupported field type ${input.type}`, `${path}.type`) + return fail('SCHEMA_INVALID', `Unsupported field type ${input.type}`, `${path}.type`) } function assertFieldLimit(count: number, path: string): void { if (count > MAX_SCHEMA_FIELDS) { - fail('INVALID_SCHEMA', `Schema exceeds ${MAX_SCHEMA_FIELDS} field descriptors`, path) + fail( + 'SCHEMA_FIELD_LIMIT_EXCEEDED', + `Schema exceeds ${MAX_SCHEMA_FIELDS} field descriptors`, + path, + ) } } @@ -149,11 +153,11 @@ function parseFields( count: { value: number }, ): SchemaFields { if (!isRecord(input) || Object.keys(input).length === 0) { - return fail('INVALID_SCHEMA', 'fields must be a non-empty object', path) + return fail('SCHEMA_INVALID', 'fields must be a non-empty object', path) } const fields = Object.create(null) as SchemaFields for (const [name, descriptor] of Object.entries(input)) { - if (!FIELD_NAME.test(name)) fail('INVALID_SCHEMA', 'Invalid field name', `${path}.${name}`) + if (!FIELD_NAME.test(name)) fail('SCHEMA_INVALID', 'Invalid field name', `${path}.${name}`) count.value += 1 assertFieldLimit(count.value, `${path}.${name}`) fields[name] = parseDescriptor(descriptor, `${path}.${name}`, depth, count) @@ -164,22 +168,22 @@ function parseFields( function parseRelation(value: unknown, path: string): string | undefined { if (value === undefined) return undefined if (typeof value !== 'string' || !SCHEMA_UID.test(value)) { - return fail('INVALID_SCHEMA', 'Expected a lowercase 32-byte schema UID', path) + return fail('SCHEMA_INVALID', 'Expected a lowercase 32-byte schema UID', path) } return value } export function parseSchema(input: unknown): SchemaDefinition { - if (!isRecord(input)) return fail('INVALID_SCHEMA', 'Schema must be an object', '$') + if (!isRecord(input)) return fail('SCHEMA_INVALID', 'Schema must be an object', '$') assertOnlyProperties(input, SCHEMA_PROPERTIES, '$') if (input.xcsVersion !== '0.1') { - return fail('INVALID_SCHEMA', 'Unsupported XCS schema version', '$.xcsVersion') + return fail('SCHEMA_INVALID', 'Unsupported XCS schema version', '$.xcsVersion') } const parent = parseRelation(input.extends, '$.extends') const predecessor = parseRelation(input.supersedes, '$.supersedes') if (parent !== undefined && parent === predecessor) { - return fail('INVALID_SCHEMA', 'extends and supersedes must reference different schemas', '$') + return fail('SCHEMA_INVALID', 'extends and supersedes must reference different schemas', '$') } return { @@ -192,6 +196,9 @@ export function parseSchema(input: unknown): SchemaDefinition { } } +/** Normative XCS v0.1 name retained for existing consumers. */ +export const validateSchema = parseSchema + export function parseSchemaBytes(bytes: Uint8Array): SchemaDefinition { return parseSchema(parseCanonicalJson(bytes)) } diff --git a/apps/indexer/src/lib/xcs/tr46.d.ts b/apps/indexer/src/lib/xcs/tr46.d.ts index 7ce46a7..1ac8b70 100644 --- a/apps/indexer/src/lib/xcs/tr46.d.ts +++ b/apps/indexer/src/lib/xcs/tr46.d.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/src/tr46.d.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/src/tr46.d.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). declare module 'tr46' { interface ToASCIIOptions { checkBidi?: boolean diff --git a/apps/indexer/src/main.ts b/apps/indexer/src/main.ts index cac7964..c8f930f 100644 --- a/apps/indexer/src/main.ts +++ b/apps/indexer/src/main.ts @@ -1,4 +1,4 @@ -import { createDatabaseClient } from './lib/db/client.js' +import { createDatabaseClient } from './lib/db/index.js' import { loadIndexerConfig } from './config.js' import { PostgresIndexerRepository } from './repository.js' diff --git a/apps/indexer/src/maintenance.ts b/apps/indexer/src/maintenance.ts index 7d4de12..58af69d 100644 --- a/apps/indexer/src/maintenance.ts +++ b/apps/indexer/src/maintenance.ts @@ -2,13 +2,14 @@ import { readFile } from 'node:fs/promises' import { parseNetworkProfile, type NetworkProfile } from './lib/xcs/index.js' import { + createDatabaseClient, credentialEvents, credentialGenerations, ledgerCheckpoints, schemaEvents, schemas, -} from '#db/schema' -import { type XcsDatabase, createDatabaseClient } from './lib/db/client.js' + type XcsDatabase, +} from './lib/db/index.js' import { count, eq } from 'drizzle-orm' import { diff --git a/apps/indexer/src/projection-digest.ts b/apps/indexer/src/projection-digest.ts index abadba2..2243c51 100644 --- a/apps/indexer/src/projection-digest.ts +++ b/apps/indexer/src/projection-digest.ts @@ -6,8 +6,8 @@ import { networkProfiles, schemaEvents, schemas, -} from '#db/schema' -import { type XcsDatabase } from './lib/db/client.js' + type XcsDatabase, +} from './lib/db/index.js' import { asc, eq } from 'drizzle-orm' import { canonicalJson, encodeUtf8, sha256Hex } from './serialization.js' diff --git a/apps/indexer/src/repository.ts b/apps/indexer/src/repository.ts index 8af43d7..ce36302 100644 --- a/apps/indexer/src/repository.ts +++ b/apps/indexer/src/repository.ts @@ -1,23 +1,21 @@ import { + acquireIndexerLease, credentialEvents, credentialGenerations, + haltIndexer as haltDatabaseIndexer, ledgerCheckpoints, + lockActiveIndexerLease, networkProfiles, + releaseIndexerLease, + renewIndexerLease, + runSerializableTransaction, schemaEvents, schemas, -} from '#db/schema' -import { type XcsDatabase } from './lib/db/client.js' -import { runSerializableTransaction } from './lib/db/transactions.js' -import { + updateIndexerStatus as updateDatabaseIndexerStatus, type AcquiredIndexerLease, type IndexerLeaseToken, - acquireIndexerLease, - haltIndexer as haltDatabaseIndexer, - lockActiveIndexerLease, - releaseIndexerLease, - renewIndexerLease, - updateIndexerStatus as updateDatabaseIndexerStatus, -} from './lib/db/indexer-fencing.js' + type XcsDatabase, +} from './lib/db/index.js' import type { JsonValue } from './lib/xcs/index.js' import { and, asc, desc, eq, isNull } from 'drizzle-orm' diff --git a/apps/indexer/src/types.ts b/apps/indexer/src/types.ts index b0f468c..f222a16 100644 --- a/apps/indexer/src/types.ts +++ b/apps/indexer/src/types.ts @@ -5,7 +5,7 @@ import type { ResolvedSchema, SchemaDefinition, } from './lib/xcs/index.js' -import type { AcquiredIndexerLease, IndexerLeaseToken } from './lib/db/indexer-fencing.js' +import type { AcquiredIndexerLease, IndexerLeaseToken } from './lib/db/index.js' export type { NetworkProfile, RegisteredSchema, ResolvedSchema, SchemaDefinition } export type { AcquiredIndexerLease, IndexerLeaseToken } diff --git a/apps/indexer/src/xrpl-source.ts b/apps/indexer/src/xrpl-source.ts index 6f35527..7e48d88 100644 --- a/apps/indexer/src/xrpl-source.ts +++ b/apps/indexer/src/xrpl-source.ts @@ -1,4 +1,6 @@ -import { Client } from 'xrpl' +import { createHash } from 'node:crypto' +import { decodeLedgerData } from 'ripple-binary-codec' +import { Client, decode, hashes } from 'xrpl' import { assertRegistryPolicy, @@ -70,8 +72,78 @@ function nonEmptyString(value: unknown, label: string): string { return value } +function binaryHex(value: unknown, label: string): string { + if (typeof value !== 'string' || !/^(?:[0-9a-fA-F]{2})+$/u.test(value)) { + return sourceFailure('SOURCE_RESPONSE_INVALID', `${label} must be binary hex`) + } + return value.toUpperCase() +} + +function decodedTransaction(envelope: Record): Record { + const blob = binaryHex(envelope.tx_blob, 'transaction blob') + const meta = binaryHex( + Object.hasOwn(envelope, 'meta_blob') ? envelope.meta_blob : envelope.meta, + 'transaction metadata blob', + ) + if ( + Object.hasOwn(envelope, 'meta_blob') && + Object.hasOwn(envelope, 'meta') && + binaryHex(envelope.meta, 'transaction metadata blob') !== meta + ) { + return sourceFailure('SOURCE_RESPONSE_INVALID', 'Binary metadata representations disagree') + } + try { + const transaction = decode(blob) + const metadata = decode(meta) + // xrpl's signed-transaction helper deliberately rejects protocol pseudo-transactions. + // Their ID uses the same TXN\0 domain, although no account signs these ledger operations. + const pseudo = ['EnableAmendment', 'SetFee', 'UNLModify'].includes( + String(transaction.TransactionType), + ) + const transactionHash = pseudo + ? createHash('sha512') + .update(Buffer.from('54584E00' + blob, 'hex')) + .digest('hex') + .slice(0, 64) + : hashes.hashSignedTx(blob) + if ( + envelope.hash !== undefined && + hash(envelope.hash, 'transaction hash') !== transactionHash.toLowerCase() + ) { + return sourceFailure('SOURCE_RESPONSE_INVALID', 'Binary transaction hash values disagree') + } + // Always decode the serialized fields: API v2 JSON can rename Amount to DeliverMax. + return { hash: transactionHash, tx_json: transaction, meta: metadata } + } catch (error) { + if (error instanceof XrplSourceError) throw error + return sourceFailure('SOURCE_RESPONSE_INVALID', 'Cannot decode binary transaction or metadata') + } +} + +function decodedLedgerHeader(envelope: Record): Record { + const blob = binaryHex(envelope.ledger_data, 'ledger header blob') + // The canonical header is 118 bytes, excluding the LWR\0 hash prefix. + if (blob.length !== 236) + return sourceFailure('SOURCE_RESPONSE_INVALID', 'Binary ledger header length is invalid') + try { + const header = asRecord(decodeLedgerData(blob), 'decoded ledger header') + return { + ...header, + ledger_hash: hashes.hashLedgerHeader( + header as unknown as Parameters[0], + ), + closed: envelope.closed, + transactions: envelope.transactions, + } + } catch (error) { + if (error instanceof XrplSourceError) throw error + return sourceFailure('SOURCE_RESPONSE_INVALID', 'Cannot decode binary ledger header') + } +} + function normalizeTransaction(value: unknown): LedgerTransaction { - const envelope = asRecord(value, 'expanded ledger transaction') + const raw = asRecord(value, 'expanded ledger transaction') + const envelope = Object.hasOwn(raw, 'tx_blob') ? decodedTransaction(raw) : raw const transaction = typeof envelope.tx_json === 'object' && envelope.tx_json !== null ? asRecord(envelope.tx_json, 'tx_json') @@ -97,10 +169,18 @@ function normalizeTransaction(value: unknown): LedgerTransaction { delete cleanTransaction.metaData delete cleanTransaction.hash + // Nodes add these API conveniences at request time; they are not hashed ledger metadata. + const cleanMetadata = { ...metadata } + delete cleanMetadata.nftoken_id + delete cleanMetadata.nftoken_ids + delete cleanMetadata.offer_id + delete cleanMetadata.mpt_issuance_id + delete cleanMetadata.delivered_amount + return { hash: transactionHash, transaction: cleanTransaction, - metadata: { ...metadata }, + metadata: cleanMetadata, transactionIndex, } } @@ -110,7 +190,10 @@ export function normalizeLedgerResponse(value: unknown): ValidatedLedger { if (result.validated !== true) { return sourceFailure('SOURCE_RESPONSE_INVALID', 'XRPL server returned a non-validated ledger') } - const ledger = asRecord(result.ledger, 'ledger') + const rawLedger = asRecord(result.ledger, 'ledger') + const ledger = Object.hasOwn(rawLedger, 'ledger_data') + ? decodedLedgerHeader(rawLedger) + : rawLedger if (ledger.closed !== true) { return sourceFailure('SOURCE_RESPONSE_INVALID', 'Validated ledger must be marked closed') } @@ -298,7 +381,9 @@ export class XrplLedgerSource implements LedgerSource { ledger_index: ledgerIndex, transactions: true, expand: true, - binary: false, + // Large expanded JSON responses can exceed provider WebSocket limits. Binary retains + // the complete ledger and canonical fields while reducing transport size; no partial retry. + binary: true, }) const ledger = normalizeLedgerResponse(result) if (ledger.ledgerIndex !== ledgerIndex) { diff --git a/apps/indexer/test/db/admin.integration.test.ts b/apps/indexer/test/db/admin.integration.test.ts new file mode 100644 index 0000000..d9c3c49 --- /dev/null +++ b/apps/indexer/test/db/admin.integration.test.ts @@ -0,0 +1,185 @@ +import { randomBytes, randomUUID } from 'node:crypto' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { createDatabaseClient, type DatabaseClient } from '../../src/lib/db/client.js' +import { bootstrapFirstAdmin } from '../../src/lib/db/app/bootstrap-admin.js' +import { provisionRuntimeDatabasePrivileges } from '../../src/lib/db/bootstrap.js' +import { migrateDatabase } from '../../src/lib/db/migrations.js' +import { createManagedRuntimeDatabaseUsers } from '../helpers/managedRuntimeUsers.js' + +const adminUrl = process.env.XCS_TEST_DATABASE_URL?.trim() || undefined +if (process.env.XCS_REQUIRE_POSTGRES_TESTS === '1' && !adminUrl) + throw new Error('XCS_TEST_DATABASE_URL is required') +let root: DatabaseClient, database: DatabaseClient, admin: DatabaseClient, notifier: DatabaseClient +let databaseName: string, + userId: string, + secondUserId: string, + organizationId: string, + decisionId: string +const passwords = { + clusterScope: 'dedicated' as const, + indexerPassword: randomBytes(32).toString('base64url'), + apiPassword: randomBytes(32).toString('base64url'), + payloadWriterPassword: randomBytes(32).toString('base64url'), + monitorPassword: randomBytes(32).toString('base64url'), + adminApplicationPassword: randomBytes(32).toString('base64url'), + notifierPassword: randomBytes(32).toString('base64url'), +} +const identity = { + issuer: 'https://identity.example.invalid', + subject: 'initial-admin', + operator: 'synthetic-test-operator', +} + +describe.skipIf(!adminUrl)('admin audit, operator bootstrap and restricted database roles', () => { + beforeAll(async () => { + root = createDatabaseClient(adminUrl!, { onNotice: () => undefined }) + databaseName = `xcs_admin_test_${randomUUID().replaceAll('-', '')}` + await root.sql`CREATE DATABASE ${root.sql(databaseName)} TEMPLATE template0` + const url = new URL(adminUrl!) + url.pathname = `/${databaseName}` + database = createDatabaseClient(url.toString(), { onNotice: () => undefined }) + await migrateDatabase(database) + await migrateDatabase(database) + const managedPasswords = await createManagedRuntimeDatabaseUsers(database, passwords) + await provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }) + url.username = 'xcs_admin_app' + url.password = managedPasswords.adminApplicationPassword + admin = createDatabaseClient(url.toString(), { onNotice: () => undefined }) + url.username = 'xcs_notifier' + url.password = managedPasswords.notifierPassword + notifier = createDatabaseClient(url.toString(), { onNotice: () => undefined }) + const users = await database.sql< + { id: string }[] + >`INSERT INTO app_users (identity_issuer, identity_subject) VALUES (${identity.issuer}, ${identity.subject}), (${identity.issuer}, 'another') RETURNING id` + userId = users[0]!.id + secondUserId = users[1]!.id + const [org] = await database.sql< + { id: string }[] + >`INSERT INTO app_organizations (responsible_user_id, name) VALUES (${secondUserId}, 'Synthetic organization') RETURNING id` + organizationId = org!.id + await database.sql`INSERT INTO app_organization_applications (organization_id, role) VALUES (${organizationId}, 'verifier')` + }, 30_000) + afterAll(async () => { + await admin?.close() + await notifier?.close() + await database?.close() + if (root && /^xcs_admin_test_[a-f0-9]{32}$/.test(databaseName)) + await root.sql`DROP DATABASE IF EXISTS ${root.sql(databaseName)} WITH (FORCE)` + await root?.close() + }) + + it('serializes initial admin bootstrap, grants once and retains exactly one audit record', async () => { + const outcomes = await Promise.all([ + bootstrapFirstAdmin(database, identity), + bootstrapFirstAdmin(database, identity), + ]) + expect(outcomes.map((x) => x.created).sort()).toEqual([false, true]) + expect(await database.sql`SELECT user_id, operator FROM app_admin_bootstrap_audit`).toEqual([ + { user_id: userId, operator: identity.operator }, + ]) + await expect( + bootstrapFirstAdmin(database, { ...identity, subject: 'another' }), + ).rejects.toThrow('ADMIN_BOOTSTRAP_ALREADY_INITIALIZED') + await expect( + bootstrapFirstAdmin(database, { ...identity, issuer: 'https://another.example.invalid' }), + ).rejects.toThrow('ADMIN_BOOTSTRAP_ACCOUNT_NOT_FOUND') + await database.sql`UPDATE app_user_roles SET revoked_at = now() WHERE user_id = ${userId} AND role = 'admin'` + await expect(bootstrapFirstAdmin(database, identity)).rejects.toThrow('ADMIN_BOOTSTRAP_REVOKED') + await database.sql`UPDATE app_user_roles SET revoked_at = NULL WHERE user_id = ${userId} AND role = 'admin'` + }) + + it('rolls back the approval and audit when its notification insert fails', async () => { + await expect( + admin.sql.begin(async (sql) => { + await sql`UPDATE app_organization_applications SET status = 'approved', revision = 1, reviewed_by = ${userId}, reviewed_at = now() WHERE organization_id = ${organizationId} AND role = 'verifier'` + const [decision] = await sql< + { id: string }[] + >`INSERT INTO app_admin_decisions (organization_id,role,actor_id,action,before_status,after_status,revision,idempotency_key,request_hash) VALUES (${organizationId},'verifier',${userId},'approve','pending','approved',1,${randomUUID()},${'a'.repeat(64)}) RETURNING id` + await sql`INSERT INTO app_admin_notifications (decision_id,recipient_user_id,status) VALUES (${decision!.id},${secondUserId},'invalid')` + }), + ).rejects.toMatchObject({ code: '23514' }) + expect( + await admin.sql`SELECT status, revision FROM app_organization_applications WHERE organization_id = ${organizationId}`, + ).toEqual([{ status: 'pending', revision: 0 }]) + expect(await admin.sql`SELECT id FROM app_admin_decisions`).toEqual([]) + }) + + it('enforces immutable audit, notification uniqueness and valid state metadata', async () => { + const key = randomUUID() + const [decision] = await admin.sql< + { id: string }[] + >`INSERT INTO app_admin_decisions (organization_id,role,actor_id,action,before_status,after_status,revision,idempotency_key,request_hash) VALUES (${organizationId},'verifier',${userId},'approve','pending','approved',1,${key},${'a'.repeat(64)}) RETURNING id` + decisionId = decision!.id + await expect( + admin.sql`INSERT INTO app_admin_decisions (organization_id,role,actor_id,action,before_status,after_status,revision,idempotency_key,request_hash) VALUES (${organizationId},'verifier',${userId},'approve','pending','approved',2,${key},${'a'.repeat(64)})`, + ).rejects.toMatchObject({ code: '23505' }) + await expect( + admin.sql`INSERT INTO app_admin_decisions (organization_id,role,actor_id,action,before_status,after_status,revision,idempotency_key,request_hash) VALUES (${organizationId},'verifier',${userId},'suspend','approved','suspended',2,${randomUUID()},${'a'.repeat(64)})`, + ).rejects.toMatchObject({ code: '23514' }) + await admin.sql`INSERT INTO app_admin_notifications (decision_id,recipient_user_id,status) VALUES (${decisionId},${secondUserId},'blocked')` + await expect( + admin.sql`INSERT INTO app_admin_notifications (decision_id,recipient_user_id,status) VALUES (${decisionId},${secondUserId},'blocked')`, + ).rejects.toMatchObject({ code: '23505' }) + await expect( + notifier.sql`UPDATE app_admin_notifications SET status = 'sending' WHERE decision_id = ${decisionId}`, + ).rejects.toMatchObject({ code: '23514' }) + await notifier.sql`UPDATE app_admin_notifications SET status = 'sending', attempts = attempts + 1, attempt_id = ${randomUUID()}, claimed_at = now(), recipient_email = 'synthetic@example.invalid' WHERE decision_id = ${decisionId}` + await notifier.sql`UPDATE app_admin_notifications SET status = 'uncertain', error_code = 'INTERRUPTED' WHERE decision_id = ${decisionId}` + expect( + await admin.sql`SELECT status, attempts FROM app_admin_notifications WHERE decision_id = ${decisionId}`, + ).toEqual([{ status: 'uncertain', attempts: 1 }]) + }) + + it('denies claims, ledger data, identity subjects, admin grants and audit modifications to runtime pools', async () => { + for (const client of [admin, notifier]) { + for (const query of [ + () => client.sql`SELECT * FROM app_credential_metadata`, + () => client.sql`SELECT * FROM app_presentations`, + () => client.sql`SELECT * FROM schemas`, + () => client.sql`SELECT identity_subject FROM public.app_users`, + () => + client.sql`INSERT INTO app_user_roles (user_id,role) VALUES (${secondUserId},'admin')`, + () => + client.sql`UPDATE app_admin_decisions SET reason = 'tampered' WHERE id = ${decisionId}`, + () => client.sql`DELETE FROM app_admin_decisions WHERE id = ${decisionId}`, + () => client.sql`DELETE FROM app_admin_notifications WHERE decision_id = ${decisionId}`, + ]) + await expect(query()).rejects.toMatchObject({ code: '42501' }) + } + await expect(notifier.sql`SELECT * FROM app_documents`).rejects.toMatchObject({ code: '42501' }) + await expect( + notifier.sql`UPDATE app_organization_applications SET revision = 2`, + ).rejects.toMatchObject({ code: '42501' }) + await expect(admin.sql`UPDATE app_admin_notifications SET attempts = 0`).rejects.toMatchObject({ + code: '42501', + }) + await expect( + admin.sql`UPDATE app_organizations SET responsible_user_id = ${userId}`, + ).rejects.toMatchObject({ code: '42501' }) + }) + + it('removes stale column grants without changing managed authentication', async () => { + const before = await database.sql< + { roleName: string; canLogin: boolean; passwordHash: string | null }[] + >` + SELECT rolname AS "roleName", rolcanlogin AS "canLogin", rolpassword AS "passwordHash" + FROM pg_authid WHERE rolname IN ('xcs_admin_app','xcs_notifier') ORDER BY rolname + ` + await database.sql`GRANT SELECT (identity_subject) ON app_users TO xcs_admin_app, xcs_notifier` + await provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }) + const after = await database.sql< + { roleName: string; canLogin: boolean; passwordHash: string | null }[] + >` + SELECT rolname AS "roleName", rolcanlogin AS "canLogin", rolpassword AS "passwordHash" + FROM pg_authid WHERE rolname IN ('xcs_admin_app','xcs_notifier') ORDER BY rolname + ` + expect(after).toEqual(before) + expect(after.every(({ canLogin }) => canLogin)).toBe(true) + for (const client of [admin, notifier]) + await expect(client.sql`SELECT identity_subject FROM public.app_users`).rejects.toMatchObject( + { + code: '42501', + }, + ) + }) +}) diff --git a/apps/indexer/test/db/app-model.integration.test.ts b/apps/indexer/test/db/app-model.integration.test.ts new file mode 100644 index 0000000..d226d23 --- /dev/null +++ b/apps/indexer/test/db/app-model.integration.test.ts @@ -0,0 +1,419 @@ +import { randomUUID } from 'node:crypto' +import { mkdtemp, mkdir, readFile, copyFile, writeFile, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { eq, sql } from 'drizzle-orm' +import { migrate } from 'drizzle-orm/postgres-js/migrator' +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { createDatabaseClient, type DatabaseClient } from '../../src/lib/db/client.js' +import { claimInvitation } from '../../src/lib/db/app/invitations.js' +import { createAppToken, hashAppToken } from '../../src/lib/db/app/tokens.js' +import { filterCredentialClaims, getCredentialAccess } from '../../src/lib/db/app/visibility.js' +import { + appUsers, + appUserRoles, + appOrganizations, + appOrganizationApplications, + appSchemaMetadata, + appInvites, + appCredentialMetadata, + appPresentations, + appWallets, + appDocuments, +} from '#db/schema/app/index.js' + +const adminUrl = process.env.XCS_TEST_DATABASE_URL?.trim() || undefined +if (process.env.XCS_REQUIRE_POSTGRES_TESTS === '1' && !adminUrl) + throw new Error('XCS_TEST_DATABASE_URL is required') +const migrationsFolder = fileURLToPath(new URL('../../../../db/migrations', import.meta.url)) +const ids = { + issuer: randomUUID(), + recipient: randomUUID(), + verifier: randomUUID(), + outsider: randomUUID(), + admin: randomUUID(), + issuerOrg: randomUUID(), + verifierOrg: randomUUID(), + otherOrg: randomUUID(), +} +const key = { profileId: 'app-testnet', generationId: 'b'.repeat(64) } +const schemaUid = 'a'.repeat(64) +const address = 'rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh' +const privateClaims = { course: 'XRPL basics', learner: { name: 'Synthetic learner' } } +let administrator: DatabaseClient +let database: DatabaseClient +let databaseName: string +let fixtureFolder: string | undefined +let presentationToken: string + +async function invitation(overrides: Partial = {}) { + const token = createAppToken() + const [invite] = await database.db + .insert(appInvites) + .values({ + organizationId: ids.issuerOrg, + profileId: key.profileId, + schemaUid, + deliveryEmail: 'invited@example.invalid', + createdBy: ids.issuer, + tokenHash: token.tokenHash, + expiresAt: new Date(Date.now() + 3600_000), + ...overrides, + }) + .returning() + return { ...token, invite: invite! } +} + +async function access(userId: string | null, token?: string, identity = key) { + return getCredentialAccess(database.db, { + ...identity, + viewerUserId: userId, + ...(token === undefined ? {} : { presentationToken: token }), + }) +} + +describe.skipIf(!adminUrl)('application model on PostgreSQL', () => { + beforeAll(async () => { + administrator = createDatabaseClient(adminUrl!) + databaseName = `xcs_app_test_${randomUUID().replaceAll('-', '')}` + await administrator.sql`CREATE DATABASE ${administrator.sql(databaseName)} TEMPLATE template0` + const url = new URL(adminUrl!) + url.pathname = `/${databaseName}` + database = createDatabaseClient(url.toString()) + + // Upgrade an actual previous database; preserve a pre-existing public payload. + const journal = JSON.parse( + await readFile(join(migrationsFolder, 'meta/_journal.json'), 'utf8'), + ) as { + version: string + dialect: string + entries: { tag: string }[] + } + const appIndex = journal.entries.findIndex((entry) => entry.tag.endsWith('_application_model')) + expect(appIndex).toBeGreaterThan(0) + fixtureFolder = await mkdtemp(join(tmpdir(), 'xcs-app-migrations-')) + await mkdir(join(fixtureFolder, 'meta')) + const entries = journal.entries.slice(0, appIndex) + for (const entry of entries) + await copyFile( + join(migrationsFolder, `${entry.tag}.sql`), + join(fixtureFolder, `${entry.tag}.sql`), + ) + await writeFile( + join(fixtureFolder, 'meta/_journal.json'), + JSON.stringify({ ...journal, entries }), + ) + await migrate(database.db, { migrationsFolder: fixtureFolder }) + await database.sql`INSERT INTO hosted_payloads (locator, digest_hex, content) VALUES (${'1'.repeat(20)}, ${'a'.repeat(64)}, ${'{"test":true}'})` + await migrate(database.db, { migrationsFolder }) + await migrate(database.db, { migrationsFolder }) + }, 30_000) + + afterAll(async () => { + await database?.close() + if (administrator && /^xcs_app_test_[a-f0-9]{32}$/.test(databaseName)) { + await administrator.sql`DROP DATABASE IF EXISTS ${administrator.sql(databaseName)} WITH (FORCE)` + } + await administrator?.close() + if (fixtureFolder?.startsWith(join(tmpdir(), 'xcs-app-migrations-'))) + await rm(fixtureFolder, { recursive: true, force: true }) + }) + + beforeEach(async () => { + // Only this suite's isolated database and application tables are reset. + await database.sql`TRUNCATE app_presentation_challenges, app_presentation_proofs, app_verifier_history, app_issuer_payloads, app_invite_deliveries, app_admin_notifications, app_admin_decisions, app_admin_bootstrap_audit, app_wallet_challenges, app_sessions, app_auth_transactions, app_presentations, app_credential_metadata, app_invites, app_schema_metadata, app_documents, app_organization_applications, app_wallets, app_user_roles, app_organizations, app_users` + await database.db.insert(appUsers).values( + Object.entries(ids) + .filter(([name]) => !name.endsWith('Org')) + .map(([name, id]) => ({ + id, + identityIssuer: 'https://identity.example.invalid', + identitySubject: name, + email: `${name}@example.invalid`, + })), + ) + await database.db.insert(appUserRoles).values({ userId: ids.admin, role: 'admin' }) + await database.db.insert(appOrganizations).values([ + { id: ids.issuerOrg, responsibleUserId: ids.issuer, name: 'Example school' }, + { id: ids.verifierOrg, responsibleUserId: ids.verifier, name: 'Example verifier' }, + { id: ids.otherOrg, responsibleUserId: ids.outsider, name: 'Other verifier' }, + ]) + await database.db.insert(appOrganizationApplications).values( + [ + { organizationId: ids.issuerOrg, role: 'issuer' as const }, + { organizationId: ids.verifierOrg, role: 'verifier' as const }, + { organizationId: ids.otherOrg, role: 'verifier' as const }, + ].map((row) => ({ + ...row, + status: 'approved' as const, + reviewedBy: ids.admin, + reviewedAt: sql`CURRENT_TIMESTAMP`, + })), + ) + await database.db.insert(appSchemaMetadata).values({ + profileId: key.profileId, + schemaUid, + organizationId: ids.issuerOrg, + registrationTransactionHash: 'c'.repeat(64), + }) + await database.db.insert(appCredentialMetadata).values({ + ...key, + schemaUid, + issuerOrganizationId: ids.issuerOrg, + recipientUserId: ids.recipient, + issuerAddress: address, + subjectAddress: address, + visibility: 'private', + publicFields: ['/course'], + payloadStorageKey: 'private/fictional-test-object', + payloadDigest: 'd'.repeat(64), + creationTransactionHash: 'e'.repeat(64), + creationLedgerIndex: 100, + }) + const token = createAppToken() + presentationToken = token.token + await database.db.insert(appPresentations).values({ + ...key, + recipientUserId: ids.recipient, + verifierOrganizationId: ids.verifierOrg, + scope: 'full', + tokenHash: token.tokenHash, + }) + }) + + it('upgrades without altering existing hosted payload bytes', async () => { + const rows = + await database.sql`SELECT content FROM hosted_payloads WHERE locator = ${'1'.repeat(20)}` + expect(rows[0]?.content).toBe('{"test":true}') + }) + + it('also creates the complete model on an empty database', async () => { + const name = `xcs_app_test_${randomUUID().replaceAll('-', '')}` + await administrator.sql`CREATE DATABASE ${administrator.sql(name)} TEMPLATE template0` + const url = new URL(adminUrl!) + url.pathname = `/${name}` + const fresh = createDatabaseClient(url.toString()) + try { + await migrate(fresh.db, { migrationsFolder }) + const [row] = await fresh.sql<{ count: number }[]>` + SELECT count(*)::integer AS count FROM information_schema.tables + WHERE table_schema = 'public' AND left(table_name, 4) = 'app_' + ` + expect(row?.count).toBe(21) + expect(await fresh.db.select().from(appUsers)).toEqual([]) + } finally { + await fresh.close() + await administrator.sql`DROP DATABASE ${administrator.sql(name)} WITH (FORCE)` + } + }) + + it.each([null, ids.outsider, ids.admin])( + 'exposes only public fields to non-owners including admin %s', + async (user) => { + const result = await access(user, presentationToken) + expect(result?.scope).toBe('public') + expect(filterCredentialClaims(privateClaims, result!)).toEqual({ course: 'XRPL basics' }) + }, + ) + + it.each([ids.issuer, ids.recipient])( + 'allows the owner without a presentation %s', + async (user) => { + const result = await access(user) + expect(result?.scope).toBe('full') + expect(filterCredentialClaims(privateClaims, result!)).toEqual(privateClaims) + }, + ) + + it('requires the designated verifier AND its grant, without an automatic expiry', async () => { + expect((await access(ids.verifier))?.scope).toBe('public') + expect((await access(ids.verifier, 'invalid'))?.scope).toBe('public') + await database.db.update(appPresentations).set({ createdAt: new Date('2000-01-01T00:00:00Z') }) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('full') + expect((await access(ids.outsider, presentationToken))?.scope).toBe('public') + }) + + it('rechecks grant revocation and organization approval on subsequent reads', async () => { + await database.db.update(appPresentations).set({ revokedAt: sql`CURRENT_TIMESTAMP` }) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('public') + await database.db.update(appPresentations).set({ revokedAt: null }) + await database.db + .update(appOrganizationApplications) + .set({ status: 'suspended', reviewReason: 'Test suspension' }) + .where(eq(appOrganizationApplications.organizationId, ids.verifierOrg)) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('public') + }) + + it('rejects suspended accounts/organizations and public-only grants for private access', async () => { + await database.db + .update(appUsers) + .set({ status: 'suspended' }) + .where(eq(appUsers.id, ids.verifier)) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('public') + await database.db + .update(appUsers) + .set({ status: 'active' }) + .where(eq(appUsers.id, ids.verifier)) + await database.db + .update(appOrganizations) + .set({ status: 'suspended' }) + .where(eq(appOrganizations.id, ids.verifierOrg)) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('public') + await database.db + .update(appOrganizations) + .set({ status: 'active' }) + .where(eq(appOrganizations.id, ids.verifierOrg)) + await database.db.update(appPresentations).set({ scope: 'public' }) + expect((await access(ids.verifier, presentationToken))?.scope).toBe('public') + }) + + it('does not reuse a grant for another generation or network profile', async () => { + expect( + await access(ids.verifier, presentationToken, { ...key, profileId: 'another-network' }), + ).toBeNull() + const [original] = await database.db.select().from(appCredentialMetadata) + const another = { ...key, generationId: 'f'.repeat(64) } + await database.db.insert(appCredentialMetadata).values({ ...original!, ...another }) + expect((await access(ids.verifier, presentationToken, another))?.scope).toBe('public') + const anotherProfile = { ...key, profileId: 'another-network' } + const [schema] = await database.db.select().from(appSchemaMetadata) + await database.db + .insert(appSchemaMetadata) + .values({ ...schema!, profileId: anotherProfile.profileId }) + await database.db.insert(appCredentialMetadata).values({ ...original!, ...anotherProfile }) + expect((await access(ids.verifier, presentationToken, anotherProfile))?.scope).toBe('public') + }) + + it('returns all public credential claims even anonymously', async () => { + await database.db.update(appCredentialMetadata).set({ visibility: 'public' }) + expect((await access(null))?.scope).toBe('full') + }) + + it('lets a different-email account claim the link without verifying that email', async () => { + const { token } = await invitation() + expect((await claimInvitation(database.db, { token, userId: ids.recipient }))?.claimedBy).toBe( + ids.recipient, + ) + const [user] = await database.db.select().from(appUsers).where(eq(appUsers.id, ids.recipient)) + expect(user?.email).toBe('recipient@example.invalid') + expect(user?.emailVerifiedAt).toBeNull() + expect(await claimInvitation(database.db, { token, userId: ids.outsider })).toBeNull() + }) + + it('allows only one concurrent claimant', async () => { + const { token } = await invitation() + const results = await Promise.all( + [ids.recipient, ids.outsider].map((userId) => + claimInvitation(database.db, { token, userId }), + ), + ) + expect(results.filter(Boolean)).toHaveLength(1) + const [row] = await database.db.select().from(appInvites) + expect(row?.claimedBy).toBe(results.find(Boolean)?.claimedBy) + }) + + it('checks invitation expiry at the claim statement, not at transaction start', async () => { + const { token, invite } = await invitation() + await database.db.transaction(async (transaction) => { + await transaction + .update(appInvites) + .set({ expiresAt: sql`CURRENT_TIMESTAMP + interval '50 milliseconds'` }) + .where(eq(appInvites.id, invite.id)) + await transaction.execute(sql`SELECT pg_sleep(0.1)`) + expect(await claimInvitation(transaction, { token, userId: ids.recipient })).toBeNull() + }) + }) + + it('refuses expired, revoked, malformed invitations and suspended claimants', async () => { + const expired = await invitation({ + createdAt: new Date(Date.now() - 7200_000), + expiresAt: new Date(Date.now() - 3600_000), + }) + const revoked = await invitation({ revokedAt: new Date() }) + for (const token of [expired.token, revoked.token, 'invalid']) { + expect(await claimInvitation(database.db, { token, userId: ids.recipient })).toBeNull() + } + const valid = await invitation() + await database.db + .update(appUsers) + .set({ status: 'suspended' }) + .where(eq(appUsers.id, ids.recipient)) + expect( + await claimInvitation(database.db, { token: valid.token, userId: ids.recipient }), + ).toBeNull() + }) + + it('enforces the invitation claimant and presentation recipient in SQL', async () => { + const { token, invite } = await invitation() + await claimInvitation(database.db, { token, userId: ids.outsider }) + await expect( + database.db.update(appCredentialMetadata).set({ inviteId: invite.id }), + ).rejects.toThrow() + await expect( + database.db.update(appPresentations).set({ recipientUserId: ids.outsider }), + ).rejects.toThrow() + await expect( + database.db.update(appPresentations).set({ verifierOrganizationId: null }), + ).rejects.toThrow() + }) + + it('enforces wallet uniqueness, organization ownership and structured public-field lists', async () => { + const wallet = { userId: ids.recipient, networkId: 1, address, verifiedAt: new Date() } + await database.db.insert(appWallets).values(wallet) + await expect( + database.db.insert(appWallets).values({ ...wallet, userId: ids.outsider }), + ).rejects.toThrow() + await database.db.insert(appWallets).values({ ...wallet, networkId: 0 }) + await expect( + database.db.update(appCredentialMetadata).set({ issuerOrganizationId: ids.otherOrg }), + ).rejects.toThrow() + await expect( + database.sql`UPDATE app_credential_metadata SET public_fields = '[1]'::jsonb`, + ).rejects.toThrow() + await expect( + database.sql`UPDATE app_credential_metadata SET visibility = 'unexpected'`, + ).rejects.toThrow() + }) + + it('keeps PII removable without cascading deletion into credential ownership', async () => { + const [document] = await database.db + .insert(appDocuments) + .values({ + organizationId: ids.issuerOrg, + applicationRole: 'issuer', + storageKey: 'review/test-object', + mimeType: 'application/pdf', + byteLength: 10, + sha256: 'a'.repeat(64), + uploadedBy: ids.issuer, + }) + .returning() + await database.db.delete(appDocuments).where(eq(appDocuments.id, document!.id)) + await expect( + database.db.delete(appUsers).where(eq(appUsers.id, ids.recipient)), + ).rejects.toThrow() + await database.db + .update(appUsers) + .set({ + status: 'deleted', + identityIssuer: null, + identitySubject: null, + email: null, + emailVerifiedAt: null, + displayName: null, + deletedAt: new Date(), + }) + .where(eq(appUsers.id, ids.recipient)) + expect((await access(ids.recipient))?.scope).toBe('public') + expect(await database.db.select().from(appCredentialMetadata)).toHaveLength(1) + }) + + it('can look up the presentation only by its hash, never a raw token column', async () => { + const [presentation] = await database.db.select().from(appPresentations) + expect(presentation?.tokenHash).toBe(hashAppToken(presentationToken)) + expect(Object.values(presentation!)).not.toContain(presentationToken) + }) +}) diff --git a/apps/indexer/test/db/app-visibility.test.ts b/apps/indexer/test/db/app-visibility.test.ts new file mode 100644 index 0000000..d511ca6 --- /dev/null +++ b/apps/indexer/test/db/app-visibility.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest' + +import { filterCredentialClaims, validatePublicFields } from '../../src/lib/db/app/visibility.js' +import { createAppToken, hashAppToken } from '../../src/lib/db/app/tokens.js' + +const claims = { + course: { title: 'XRPL basics', grade: 18 }, + learner: { name: 'Synthetic learner', email: 'learner@example.invalid' }, + passed: true, + credits: 0, + optional: null, + modules: [{ name: 'Ledger', privateNote: 'Do not publish' }], + 'a/b': { '~result': 'ok' }, +} + +describe('credential claim disclosure', () => { + it('projects only named fields, including nested and escaped JSON pointers', () => { + expect( + filterCredentialClaims(claims, { + scope: 'public', + publicFields: ['/course/title', '/passed', '/credits', '/optional', '/a~1b/~0result'], + }), + ).toEqual({ + course: { title: 'XRPL basics' }, + passed: true, + credits: 0, + optional: null, + 'a/b': { '~result': 'ok' }, + }) + }) + + it('does not expand an empty or missing selection into full claims', () => { + expect(filterCredentialClaims(claims, { scope: 'public', publicFields: [] })).toEqual({}) + expect( + filterCredentialClaims(claims, { + scope: 'public', + publicFields: ['/missing', '/course/missing', '/passed/child'], + }), + ).toEqual({}) + }) + + it('copies full access and explicitly selected objects without mutating the payload', () => { + const full = filterCredentialClaims(claims, { scope: 'full', publicFields: [] }) + expect(full).toEqual(claims) + expect(full).not.toBe(claims) + expect(full.course).not.toBe(claims.course) + expect(filterCredentialClaims(claims, { scope: 'public', publicFields: ['/course'] })).toEqual({ + course: claims.course, + }) + }) + + it('treats arrays atomically and never exposes their siblings through an index selection', () => { + expect( + filterCredentialClaims(claims, { scope: 'public', publicFields: ['/modules/0/name'] }), + ).toEqual({}) + expect(filterCredentialClaims(claims, { scope: 'public', publicFields: ['/modules'] })).toEqual( + { modules: claims.modules }, + ) + }) + + it.each([ + '', + 'course.title', + '/~2', + '/__proto__/polluted', + '/constructor/prototype', + '/course/prototype', + '/' + 'x'.repeat(1024), + ])('fails closed for invalid path %s', (path) => { + expect(() => filterCredentialClaims(claims, { scope: 'public', publicFields: [path] })).toThrow( + 'INVALID_PUBLIC_FIELD_PATH', + ) + }) + + it('ignores inherited fields and bounds selector size/depth', () => { + expect(filterCredentialClaims({}, { scope: 'public', publicFields: ['/toString'] })).toEqual({}) + expect(() => validatePublicFields(Array(257).fill('/course'))).toThrow('INVALID_PUBLIC_FIELDS') + expect(() => validatePublicFields(['/' + Array(33).fill('x').join('/')])).toThrow( + 'INVALID_PUBLIC_FIELD_PATH', + ) + expect(Object.prototype).not.toHaveProperty('polluted') + }) +}) + +describe('application bearer tokens', () => { + it('uses independent 256-bit tokens and persists only their SHA-256 hashes', () => { + const first = createAppToken() + const second = createAppToken() + expect(first.token).toMatch(/^[A-Za-z0-9_-]{43}$/) + expect(first.tokenHash).toMatch(/^[0-9a-f]{64}$/) + expect(hashAppToken(first.token)).toBe(first.tokenHash) + expect(first.token).not.toBe(second.token) + expect(first.tokenHash).not.toBe(second.tokenHash) + }) + + it('rejects malformed and unbounded tokens before database access', () => { + for (const token of ['', 'bad token', 'a'.repeat(44), 'a'.repeat(10000)]) { + expect(hashAppToken(token)).toBeNull() + } + }) +}) diff --git a/apps/indexer/test/db/auth.integration.test.ts b/apps/indexer/test/db/auth.integration.test.ts new file mode 100644 index 0000000..5059c31 --- /dev/null +++ b/apps/indexer/test/db/auth.integration.test.ts @@ -0,0 +1,229 @@ +import { randomBytes, randomUUID } from 'node:crypto' +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterAll, beforeAll, describe, expect, it } from 'vitest' + +import { createDatabaseClient, type DatabaseClient } from '../../src/lib/db/client.js' +import { provisionRuntimeDatabasePrivileges } from '../../src/lib/db/bootstrap.js' +import { DATABASE_MIGRATIONS_FOLDER, migrateDatabase } from '../../src/lib/db/migrations.js' +import { createManagedRuntimeDatabaseUsers } from '../helpers/managedRuntimeUsers.js' + +const adminUrl = process.env.XCS_TEST_DATABASE_URL?.trim() || undefined +if (process.env.XCS_REQUIRE_POSTGRES_TESTS === '1' && !adminUrl) { + throw new Error('XCS_TEST_DATABASE_URL is required') +} +const passwords = { + clusterScope: 'dedicated' as const, + indexerPassword: randomBytes(32).toString('base64url'), + apiPassword: randomBytes(32).toString('base64url'), + payloadWriterPassword: randomBytes(32).toString('base64url'), + monitorPassword: randomBytes(32).toString('base64url'), + applicationPassword: randomBytes(32).toString('base64url'), +} +const address = 'rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh' +let administrator: DatabaseClient +let database: DatabaseClient +let application: DatabaseClient +let databaseName: string +let fixtureFolder: string | undefined +let userId: string + +async function session() { + const [row] = await application.sql<{ id: string }[]>` + INSERT INTO app_sessions (token_hash, user_id, csrf_token, expires_at, absolute_expires_at) + VALUES (${randomBytes(32).toString('hex')}, ${userId}, ${randomBytes(32).toString('base64url')}, now() + interval '1 hour', now() + interval '1 day') RETURNING id + ` + return row!.id +} + +describe.skipIf(!adminUrl)('authentication schema and restricted application role', () => { + beforeAll(async () => { + administrator = createDatabaseClient(adminUrl!, { onNotice: () => undefined }) + databaseName = `xcs_auth_test_${randomUUID().replaceAll('-', '')}` + await administrator.sql`CREATE DATABASE ${administrator.sql(databaseName)} TEMPLATE template0` + const url = new URL(adminUrl!) + url.pathname = `/${databaseName}` + database = createDatabaseClient(url.toString(), { onNotice: () => undefined }) + + // Upgrade the actual preceding catalog, retaining an existing identity/admin grant. + const journal = JSON.parse( + await readFile(join(DATABASE_MIGRATIONS_FOLDER, 'meta/_journal.json'), 'utf8'), + ) as { entries: { tag: string }[] } + const authIndex = journal.entries.findIndex(({ tag }) => tag === '0004_auth_sessions') + expect(authIndex).toBe(4) + fixtureFolder = await mkdtemp(join(tmpdir(), 'xcs-auth-migrations-')) + await mkdir(join(fixtureFolder, 'meta')) + const entries = journal.entries.slice(0, authIndex) + for (const { tag } of entries) + await copyFile( + join(DATABASE_MIGRATIONS_FOLDER, `${tag}.sql`), + join(fixtureFolder, `${tag}.sql`), + ) + await writeFile( + join(fixtureFolder, 'meta/_journal.json'), + JSON.stringify({ ...journal, entries }), + ) + await migrateDatabase(database, fixtureFolder) + const [user] = await database.sql<{ id: string }[]>` + INSERT INTO app_users (identity_issuer, identity_subject) VALUES ('https://identity.example.invalid', 'existing') RETURNING id + ` + userId = user!.id + await database.sql`INSERT INTO app_user_roles (user_id, role) VALUES (${userId}, 'admin')` + await migrateDatabase(database) + await migrateDatabase(database) + const managedPasswords = await createManagedRuntimeDatabaseUsers(database, passwords) + await provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }) + url.username = 'xcs_app' + url.password = managedPasswords.applicationPassword + application = createDatabaseClient(url.toString(), { onNotice: () => undefined }) + }, 30_000) + + afterAll(async () => { + await application?.close() + await database?.close() + if (administrator && /^xcs_auth_test_[a-f0-9]{32}$/.test(databaseName)) { + await administrator.sql`DROP DATABASE IF EXISTS ${administrator.sql(databaseName)} WITH (FORCE)` + } + await administrator?.close() + if (fixtureFolder) await rm(fixtureFolder, { recursive: true, force: true }) + }) + + it('preserves existing identities and admin grants while defaulting new roles to recipient', async () => { + expect( + await application.sql`SELECT role FROM app_user_roles WHERE user_id = ${userId}`, + ).toEqual([{ role: 'admin' }]) + const [user] = await application.sql<{ id: string }[]>` + INSERT INTO app_users (identity_issuer, identity_subject, email, display_name) + VALUES ('https://identity.example.invalid', 'new', 'fixture@example.invalid', 'Fixture') RETURNING id + ` + await application.sql`INSERT INTO app_user_roles (user_id) VALUES (${user!.id})` + expect( + await application.sql`SELECT role FROM app_user_roles WHERE user_id = ${user!.id}`, + ).toEqual([{ role: 'recipient' }]) + await application.sql`UPDATE app_users SET display_name = 'Updated fixture' WHERE id = ${user!.id}` + for (const query of [ + () => + application.sql`INSERT INTO app_user_roles (user_id, role) VALUES (${user!.id}, 'admin')`, + () => application.sql`UPDATE app_user_roles SET role = 'admin' WHERE user_id = ${user!.id}`, + () => application.sql`UPDATE app_users SET status = 'active' WHERE id = ${user!.id}`, + () => + application.sql`UPDATE app_users SET identity_subject = 'another' WHERE id = ${user!.id}`, + () => application.sql`DELETE FROM app_users WHERE id = ${user!.id}`, + () => application.sql`UPDATE app_organization_applications SET status = 'approved'`, + () => application.sql`SELECT * FROM app_organization_applications`, + () => application.sql`SELECT * FROM app_credential_metadata`, + () => application.sql`SELECT * FROM app_documents`, + () => application.sql`SELECT * FROM app_presentations`, + () => application.sql`SELECT * FROM hosted_payloads`, + () => application.sql`DELETE FROM schemas`, + () => application.sql`CREATE TABLE forbidden_auth_table (id integer)`, + ]) + await expect(query()).rejects.toMatchObject({ code: '42501' }) + expect( + await application.sql`SELECT organization_id, role, status FROM app_organization_applications`, + ).toEqual([]) + }) + + it('checks session bounds and makes challenge consumption atomic with cascade on logout', async () => { + const sessionId = await session() + await expect( + application.sql`UPDATE app_sessions SET expires_at = absolute_expires_at + interval '1 second' WHERE id = ${sessionId}`, + ).rejects.toMatchObject({ code: '23514' }) + await expect( + application.sql`UPDATE app_sessions SET token_hash = 'invalid' WHERE id = ${sessionId}`, + ).rejects.toMatchObject({ code: '23514' }) + await expect( + application.sql`UPDATE app_sessions SET absolute_expires_at = 'infinity' WHERE id = ${sessionId}`, + ).rejects.toMatchObject({ code: '23514' }) + const [challenge] = await application.sql<{ id: string }[]>` + INSERT INTO app_wallet_challenges (session_id, network_id, address, message, expires_at) + VALUES (${sessionId}, 1, ${address}, 'Synthetic proof fixture', now() + interval '5 minutes') RETURNING id + ` + const consumed = await Promise.all( + [1, 2].map( + () => application.sql` + DELETE FROM app_wallet_challenges WHERE id = ${challenge!.id} AND session_id = ${sessionId} AND expires_at > now() RETURNING id + `, + ), + ) + expect(consumed.map((rows) => rows.length).sort()).toEqual([0, 1]) + await application.sql` + INSERT INTO app_wallet_challenges (session_id, network_id, address, message, expires_at) + VALUES (${sessionId}, 1, ${address}, 'Synthetic proof fixture', now() + interval '5 minutes') + ` + await application.sql`DELETE FROM app_sessions WHERE id = ${sessionId}` + expect( + await application.sql`SELECT id FROM app_wallet_challenges WHERE session_id = ${sessionId}`, + ).toEqual([]) + }) + + it('enforces transaction shape, expiry and one-time state consumption', async () => { + const stateHash = randomBytes(32).toString('hex') + await application.sql` + INSERT INTO app_auth_transactions (state_hash, browser_hash, nonce, code_verifier, return_to, expires_at) + VALUES (${stateHash}, ${randomBytes(32).toString('hex')}, ${randomBytes(32).toString('base64url')}, ${randomBytes(32).toString('base64url')}, '/account', now() + interval '5 minutes') + ` + for (const invalid of [ + '//attacker.example', + '/\\attacker.example', + 'https://attacker.example', + '/account\n', + ]) { + await expect( + application.sql`UPDATE app_auth_transactions SET return_to = ${invalid} WHERE state_hash = ${stateHash}`, + ).rejects.toMatchObject({ code: '23514' }) + } + await expect( + application.sql`UPDATE app_auth_transactions SET expires_at = created_at WHERE state_hash = ${stateHash}`, + ).rejects.toMatchObject({ code: '23514' }) + const results = await Promise.all( + [1, 2].map( + () => + application.sql`DELETE FROM app_auth_transactions WHERE state_hash = ${stateHash} RETURNING state_hash`, + ), + ) + expect(results.map((rows) => rows.length).sort()).toEqual([0, 1]) + }) + + it('permits wallet verification without allowing account reassignment', async () => { + await application.sql`INSERT INTO app_wallets (user_id, network_id, address, verified_at) VALUES (${userId}, 1, ${address}, now())` + await application.sql`UPDATE app_wallets SET revoked_at = now() WHERE user_id = ${userId}` + await expect( + application.sql`UPDATE app_wallets SET user_id = ${userId} WHERE address = ${address}`, + ).rejects.toMatchObject({ code: '42501' }) + await expect( + application.sql`INSERT INTO app_wallets (user_id, network_id, address, verified_at) VALUES (${userId}, 4294967296, ${address}, now())`, + ).rejects.toMatchObject({ code: '23514' }) + await expect( + application.sql`INSERT INTO app_wallets (user_id, network_id, address, verified_at) VALUES (${userId}, 1, ${address}, now())`, + ).rejects.toMatchObject({ code: '23505' }) + }) + + it('normalizes accidental column grants without changing managed authentication', async () => { + const [before] = await database.sql<{ canLogin: boolean; passwordHash: string | null }[]>` + SELECT rolcanlogin AS "canLogin", rolpassword AS "passwordHash" + FROM pg_authid WHERE rolname = 'xcs_app' + ` + await database.sql`GRANT UPDATE (status) ON app_users TO xcs_app` + await provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }) + + const [after] = await database.sql<{ canLogin: boolean; passwordHash: string | null }[]>` + SELECT rolcanlogin AS "canLogin", rolpassword AS "passwordHash" + FROM pg_authid WHERE rolname = 'xcs_app' + ` + expect(after).toEqual(before) + expect(after?.canLogin).toBe(true) + await expect( + application.sql`UPDATE app_users SET status = 'active' WHERE id = ${userId}`, + ).rejects.toMatchObject({ code: '42501' }) + expect(await application.sql`SELECT id FROM public.app_users WHERE id = ${userId}`).toEqual([ + { id: userId }, + ]) + const [grant] = await database.sql< + { insertAllowed: boolean }[] + >`SELECT has_column_privilege('xcs_app', 'app_user_roles', 'user_id', 'INSERT') AS "insertAllowed"` + expect(grant?.insertAllowed).toBe(true) + }) +}) diff --git a/apps/indexer/test/db/environment.test.ts b/apps/indexer/test/db/environment.test.ts new file mode 100644 index 0000000..090ddcd --- /dev/null +++ b/apps/indexer/test/db/environment.test.ts @@ -0,0 +1,49 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { requiredEnvironment } from '../../src/lib/db/bin/environment.js' + +const directories: string[] = [] +afterEach(() => { + vi.unstubAllEnvs() + for (const path of directories.splice(0)) rmSync(path, { recursive: true, force: true }) +}) + +function fixture(value: string): string { + const directory = mkdtempSync(join(tmpdir(), 'xcs-db-env-')) + directories.push(directory) + const path = join(directory, 'secret') + writeFileSync(path, value) + return path +} + +describe('database command environment', () => { + it('uses a direct URL or a one-line secret file without changing encoded credentials', () => { + const value = 'postgres://test:fixture%25encoded@127.0.0.1/example?sslmode=verify-full' + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL_FILE', undefined) + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL', value) + expect(requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL')).toBe(value) + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL', undefined) + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL_FILE', fixture(`${value}\r\n`)) + expect(requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL')).toBe(value) + }) + + it('rejects missing, conflicting, empty, multiline and oversized values', () => { + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL', undefined) + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL_FILE', undefined) + expect(() => requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL')).toThrow( + 'DATABASE_ENVIRONMENT_REQUIRED', + ) + for (const value of ['', '\n', 'first\nsecond', 'x'.repeat(16_385)]) { + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL_FILE', fixture(value)) + expect(() => requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL')).toThrow() + } + vi.stubEnv('XCS_BOOTSTRAP_DATABASE_URL', 'fixture-direct-value') + expect(() => requiredEnvironment('XCS_BOOTSTRAP_DATABASE_URL')).toThrow( + 'DATABASE_ENVIRONMENT_CONFLICT', + ) + }) +}) diff --git a/apps/indexer/test/db/migrations.integration.test.ts b/apps/indexer/test/db/migrations.integration.test.ts new file mode 100644 index 0000000..e025b3b --- /dev/null +++ b/apps/indexer/test/db/migrations.integration.test.ts @@ -0,0 +1,527 @@ +import { randomUUID } from 'node:crypto' +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import type { TransactionSql } from 'postgres' +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' + +import { bootstrapDatabase } from '../../src/lib/db/bootstrap.js' +import { createDatabaseClient, type DatabaseClient } from '../../src/lib/db/client.js' +import { + DATABASE_MIGRATIONS_FOLDER, + databaseMigrationStatus, + migrateDatabase, +} from '../../src/lib/db/migrations.js' +import { createManagedRuntimeDatabaseUsers } from '../helpers/managedRuntimeUsers.js' + +const adminUrl = process.env.XCS_TEST_DATABASE_URL?.trim() || undefined +if (process.env.XCS_REQUIRE_POSTGRES_TESTS === '1' && adminUrl === undefined) { + throw new Error('XCS_TEST_DATABASE_URL is required by test:postgres') +} + +interface JournalEntry { + idx: number + version: string + when: number + tag: string + breakpoints: boolean +} + +const journal = JSON.parse( + await readFile(join(DATABASE_MIGRATIONS_FOLDER, 'meta/_journal.json'), 'utf8'), +) as { version: string; dialect: string; entries: JournalEntry[] } +const tags = journal.entries.map((entry) => entry.tag) +const temporaryNames = /^xcs_migration_(?:db|role)_[0-9a-f]{32}$/u +let administrator: DatabaseClient +let database: DatabaseClient +let databaseName: string | undefined +let roleName: string | undefined +let managedDatabase: DatabaseClient | undefined +let managedRuntimeRolesCreated = false +let managedPreexistingRuntimeRoles = false +const fixtureFolders: string[] = [] +const managedProvisionerPassword = 'managed-provisioner-password-000001' + +async function fixture(count = journal.entries.length): Promise { + const folder = await mkdtemp(join(tmpdir(), 'xcs-migration-fixture-')) + fixtureFolders.push(folder) + await mkdir(join(folder, 'meta')) + const entries = journal.entries.slice(0, count) + for (const entry of entries) { + await copyFile( + join(DATABASE_MIGRATIONS_FOLDER, `${entry.tag}.sql`), + join(folder, `${entry.tag}.sql`), + ) + } + await writeFile(join(folder, 'meta/_journal.json'), JSON.stringify({ ...journal, entries })) + return folder +} + +async function additiveFixture(sql: string): Promise { + const folder = await fixture() + const entry = { + idx: journal.entries.length, + version: '7', + when: journal.entries.at(-1)!.when + 1, + tag: `${String(journal.entries.length).padStart(4, '0')}_integration_probe`, + breakpoints: true, + } + await writeFile(join(folder, `${entry.tag}.sql`), sql) + await writeFile( + join(folder, 'meta/_journal.json'), + JSON.stringify({ ...journal, entries: [...journal.entries, entry] }), + ) + return folder +} + +async function journalRows() { + return database.sql`SELECT id, hash, created_at FROM drizzle.__drizzle_migrations ORDER BY id` +} + +async function exists(relation: string): Promise { + const [row] = await database.sql<{ exists: boolean }[]>` + SELECT to_regclass(${relation}) IS NOT NULL AS exists + ` + return row?.exists === true +} + +describe.skipIf(adminUrl === undefined)('PostgreSQL 18 migration lifecycle', () => { + beforeAll(async () => { + administrator = createDatabaseClient(adminUrl!) + const [version] = await administrator.sql<{ version: number }[]>` + SELECT current_setting('server_version_num')::integer AS version + ` + expect(Math.trunc((version?.version ?? 0) / 10_000)).toBe(18) + }) + + beforeEach(async () => { + databaseName = `xcs_migration_db_${randomUUID().replaceAll('-', '')}` + if (!temporaryNames.test(databaseName)) throw new Error('Invalid test database name') + await administrator.sql`CREATE DATABASE ${administrator.sql(databaseName)} TEMPLATE template0` + const url = new URL(adminUrl!) + url.pathname = `/${databaseName}` + database = createDatabaseClient(url.toString()) + }) + + afterEach(async () => { + await managedDatabase?.close() + managedDatabase = undefined + await database?.close() + if (databaseName !== undefined && temporaryNames.test(databaseName)) { + await administrator.sql`DROP DATABASE IF EXISTS ${administrator.sql(databaseName)} WITH (FORCE)` + databaseName = undefined + } + if (managedPreexistingRuntimeRoles && roleName !== undefined && temporaryNames.test(roleName)) { + await administrator.sql` + REVOKE pg_monitor FROM xcs_monitor + GRANTED BY ${administrator.sql(roleName)} CASCADE + ` + await administrator.sql` + GRANT pg_monitor TO xcs_monitor + WITH ADMIN FALSE, SET FALSE, INHERIT TRUE + ` + await administrator.sql`REVOKE pg_monitor FROM ${administrator.sql(roleName)}` + managedPreexistingRuntimeRoles = false + } + if (managedRuntimeRolesCreated) { + await administrator.sql` + DROP ROLE IF EXISTS + xcs_indexer, + xcs_api, + xcs_payload_writer, + xcs_monitor, + xcs_app, + xcs_admin_app, + xcs_notifier, + xcs_issuer + ` + managedRuntimeRolesCreated = false + } + if (roleName !== undefined && temporaryNames.test(roleName)) { + await administrator.sql`DROP ROLE IF EXISTS ${administrator.sql(roleName)}` + roleName = undefined + } + for (const folder of fixtureFolders.splice(0)) { + await rm(folder, { recursive: true, force: true }) + } + }, 30_000) + + afterAll(async () => { + await administrator?.close() + }) + + it('inspects an empty database without DDL and applies the journal exactly once', async () => { + expect(await databaseMigrationStatus(database)).toEqual({ applied: [], pending: tags }) + const schemas = await database.sql`SELECT nspname FROM pg_namespace WHERE nspname = 'drizzle'` + expect(schemas).toHaveLength(0) + + expect(await migrateDatabase(database)).toEqual({ applied: tags, pending: [] }) + const rows = await journalRows() + expect(rows).toHaveLength(tags.length) + expect(await migrateDatabase(database)).toEqual({ applied: tags, pending: [] }) + expect(await journalRows()).toEqual(rows) + }) + + it('bootstraps and reruns atomically through a non-superuser managed provisioner', async () => { + roleName = `xcs_migration_role_${randomUUID().replaceAll('-', '')}` + if (!temporaryNames.test(roleName)) throw new Error('Invalid test role name') + const existingRoles = await administrator.sql>` + SELECT rolname FROM pg_roles + WHERE rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) + ORDER BY rolname + ` + expect([0, 8]).toContain(existingRoles.length) + const createsFreshRuntimeRoles = existingRoles.length === 0 + managedPreexistingRuntimeRoles = !createsFreshRuntimeRoles + + await administrator.sql.begin(async (transaction) => { + await transaction` + CREATE ROLE ${transaction(roleName!)} + LOGIN NOSUPERUSER CREATEDB CREATEROLE REPLICATION BYPASSRLS + ` + await transaction`SELECT set_config('xcs.test_provisioner_password', ${managedProvisionerPassword}, true)` + await transaction`SELECT set_config('xcs.test_provisioner_role', ${roleName!}, true)` + await transaction.unsafe(` + DO $xcs_test_password$ + BEGIN + EXECUTE format( + 'ALTER ROLE %I PASSWORD %L', + current_setting('xcs.test_provisioner_role'), + current_setting('xcs.test_provisioner_password') + ); + END + $xcs_test_password$; + `) + }) + await administrator.sql` + GRANT pg_monitor TO ${administrator.sql(roleName)} + WITH ADMIN TRUE, SET FALSE, INHERIT FALSE + ` + if (createsFreshRuntimeRoles) { + await createManagedRuntimeDatabaseUsers(administrator) + managedRuntimeRolesCreated = true + } + await administrator.sql` + ALTER DATABASE ${administrator.sql(databaseName!)} OWNER TO ${administrator.sql(roleName)} + ` + const managedUrl = new URL(adminUrl!) + managedUrl.pathname = `/${databaseName}` + managedUrl.username = roleName + managedUrl.password = managedProvisionerPassword + managedDatabase = createDatabaseClient(managedUrl.toString()) + + const provisioning = { clusterScope: 'dedicated' } as const + await bootstrapDatabase(managedDatabase, provisioning) + const firstJournal = await managedDatabase.sql` + SELECT id, hash, created_at FROM drizzle.__drizzle_migrations ORDER BY id + ` + expect(firstJournal).toHaveLength(tags.length) + await bootstrapDatabase(managedDatabase, provisioning) + expect( + await managedDatabase.sql` + SELECT id, hash, created_at FROM drizzle.__drizzle_migrations ORDER BY id + `, + ).toEqual(firstJournal) + + const [provisioner] = await administrator.sql< + Array<{ isSuperuser: boolean; canCreateRole: boolean }> + >` + SELECT rolsuper AS "isSuperuser", rolcreaterole AS "canCreateRole" + FROM pg_roles WHERE rolname = ${roleName} + ` + expect(provisioner).toEqual({ isSuperuser: false, canCreateRole: true }) + const runtimeRoles = await administrator.sql< + Array<{ + roleName: string + isSuperuser: boolean + canCreateDatabase: boolean + canCreateRole: boolean + canReplicate: boolean + canBypassRls: boolean + }> + >` + SELECT + rolname AS "roleName", + rolsuper AS "isSuperuser", + rolcreatedb AS "canCreateDatabase", + rolcreaterole AS "canCreateRole", + rolreplication AS "canReplicate", + rolbypassrls AS "canBypassRls" + FROM pg_roles + WHERE rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) + ORDER BY rolname + ` + expect(runtimeRoles).toHaveLength(8) + expect( + runtimeRoles.every( + (role) => + !role.isSuperuser && + !role.canCreateDatabase && + !role.canCreateRole && + !role.canReplicate && + !role.canBypassRls, + ), + ).toBe(true) + const delegations = await administrator.sql< + Array<{ adminOption: boolean; inheritOption: boolean; setOption: boolean }> + >` + SELECT + membership.admin_option AS "adminOption", + membership.inherit_option AS "inheritOption", + membership.set_option AS "setOption" + FROM pg_auth_members membership + JOIN pg_roles granted_role ON granted_role.oid = membership.roleid + JOIN pg_roles member_role ON member_role.oid = membership.member + WHERE granted_role.rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) + AND member_role.rolname = ${roleName} + ` + // Provider-created users do not delegate ADMIN OPTION to the provisioner. + // A grants-only bootstrap must therefore remain idempotent without it. + expect(delegations).toEqual([]) + }, 30_000) + + it('rejects an obsolete migration history before creating schema objects or runtime roles', async () => { + const runtimeRolesBefore = await administrator.sql` + SELECT rolname FROM pg_roles + WHERE rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) + ORDER BY rolname + ` + await database.sql`CREATE SCHEMA drizzle` + await database.sql` + CREATE TABLE drizzle.__drizzle_migrations ( + id SERIAL PRIMARY KEY, + hash text NOT NULL, + created_at bigint + ) + ` + await database.sql` + INSERT INTO drizzle.__drizzle_migrations (hash, created_at) + VALUES ('obsolete-xcs-protocol-history', 1) + ` + + await expect(migrateDatabase(database)).rejects.toThrow('DATABASE_MIGRATION_HISTORY_MISMATCH') + expect(await exists('public.network_profiles')).toBe(false) + expect( + await administrator.sql` + SELECT rolname FROM pg_roles + WHERE rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) + ORDER BY rolname + `, + ).toEqual(runtimeRolesBefore) + expect(await journalRows()).toEqual([ + { id: 1, hash: 'obsolete-xcs-protocol-history', created_at: '1' }, + ]) + }) + + it('upgrades a populated baseline and preserves legacy payload bytes and locators', async () => { + await migrateDatabase(database, await fixture(1)) + await database.sql` + INSERT INTO network_profiles ( + profile_id, xcs_version, network_id, required_amendment, registry_address, + registration_amount_drops, activation_ledger_index, activation_ledger_hash + ) VALUES ( + 'migration-test', '0.1', 1, 'Credentials', 'rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh', + 1, 100, ${'a'.repeat(64)} + ) + ` + await migrateDatabase(database, await fixture(2)) + await database.sql` + INSERT INTO hosted_payloads (locator, digest_hex, content) + VALUES (${'b'.repeat(20)}, ${'b'.repeat(64)}, '{"retained":true}') + ` + expect(await databaseMigrationStatus(database)).toEqual({ + applied: tags.slice(0, 2), + pending: tags.slice(2), + }) + await migrateDatabase(database) + expect(await database.sql`SELECT profile_id FROM network_profiles`).toEqual([ + { profile_id: 'migration-test' }, + ]) + await database.sql` + INSERT INTO hosted_payloads (locator, digest_hex, content) + VALUES (${'b'.repeat(18)}, ${'b'.repeat(64)}, '{"retained":true}') + ` + expect( + await database.sql`SELECT locator, content FROM hosted_payloads ORDER BY locator`, + ).toEqual([ + { locator: 'b'.repeat(18), content: '{"retained":true}' }, + { locator: 'b'.repeat(20), content: '{"retained":true}' }, + ]) + }) + + it('upgrades 0006 without rewriting accounts or existing presentation columns', async () => { + await migrateDatabase(database, await fixture(7)) + await database.sql`INSERT INTO app_users(identity_issuer,identity_subject,display_name) VALUES ('https://identity.test','migration-recipient','Retained recipient')` + const users = await database.sql`SELECT * FROM app_users` + const columns = + await database.sql`SELECT column_name,data_type FROM information_schema.columns WHERE table_name='app_presentations' ORDER BY ordinal_position` + const history = await journalRows() + expect(await exists('public.app_verifier_history')).toBe(false) + await migrateDatabase(database) + expect(await database.sql`SELECT * FROM app_users`).toEqual(users) + expect( + await database.sql`SELECT column_name,data_type FROM information_schema.columns WHERE table_name='app_presentations' ORDER BY ordinal_position`, + ).toEqual(columns) + expect((await journalRows()).slice(0, history.length)).toEqual(history) + expect(await exists('public.app_verifier_history')).toBe(true) + expect(await database.sql`SELECT * FROM app_verifier_history`).toEqual([]) + }) + + it('upgrades 0007 additively while preserving legacy unsigned presentations and session records', async () => { + await migrateDatabase(database, await fixture(8)) + await database.sql`INSERT INTO app_users(identity_issuer,identity_subject,display_name) VALUES ('https://identity.test','proof-migration','Retained recipient')` + const users = await database.sql`SELECT * FROM app_users` + const columns = + await database.sql`SELECT column_name,data_type FROM information_schema.columns WHERE table_name='app_presentations' ORDER BY ordinal_position` + const history = await journalRows() + expect(await exists('public.app_presentation_challenges')).toBe(false) + await migrateDatabase(database) + expect(await database.sql`SELECT * FROM app_users`).toEqual(users) + expect( + await database.sql`SELECT column_name,data_type FROM information_schema.columns WHERE table_name='app_presentations' ORDER BY ordinal_position`, + ).toEqual(columns) + expect((await journalRows()).slice(0, history.length)).toEqual(history) + expect(await exists('public.app_presentation_challenges')).toBe(true) + expect(await database.sql`SELECT * FROM app_presentation_proofs`).toEqual([]) + }) + + it.each(['tampered', 'future', 'nonprefix'] as const)( + 'rejects %s history before executing pending DDL', + async (variant) => { + await migrateDatabase(database) + const folder = await additiveFixture('CREATE TABLE migration_probe (id integer);') + if (variant === 'tampered') { + await database.sql`UPDATE drizzle.__drizzle_migrations SET hash = ${'0'.repeat(64)} WHERE id = 1` + } else if (variant === 'future') { + await database.sql` + INSERT INTO drizzle.__drizzle_migrations (hash, created_at) + VALUES (${'f'.repeat(64)}, ${journal.entries.at(-1)!.when + 10}) + ` + } else { + await database.sql`DELETE FROM drizzle.__drizzle_migrations WHERE id = 1` + } + const before = await journalRows() + await expect(databaseMigrationStatus(database, folder)).rejects.toThrow( + 'DATABASE_MIGRATION_HISTORY_MISMATCH', + ) + await expect(migrateDatabase(database, folder)).rejects.toThrow( + 'DATABASE_MIGRATION_HISTORY_MISMATCH', + ) + expect(await exists('public.migration_probe')).toBe(false) + expect(await journalRows()).toEqual(before) + }, + ) + + it('rejects an edited applied SQL file before executing a pending migration', async () => { + await migrateDatabase(database) + const before = await journalRows() + const folder = await additiveFixture('CREATE TABLE migration_probe (id integer);') + const appliedFile = join(folder, `${tags[0]}.sql`) + await writeFile(appliedFile, `${await readFile(appliedFile, 'utf8')}\n-- modified artifact\n`) + + await expect(migrateDatabase(database, folder)).rejects.toThrow( + 'DATABASE_MIGRATION_HISTORY_MISMATCH', + ) + expect(await exists('public.migration_probe')).toBe(false) + expect(await journalRows()).toEqual(before) + }) + + it.each(['index', 'timestamp', 'order'] as const)( + 'rejects an invalid catalog %s without creating the journal', + async (variant) => { + const folder = await fixture() + const entries = journal.entries.map((entry) => ({ ...entry })) + if (variant === 'index') entries[0]!.idx = 1 + else if (variant === 'timestamp') entries[0]!.when = 0.5 + else entries[1]!.when = entries[0]!.when + await writeFile(join(folder, 'meta/_journal.json'), JSON.stringify({ ...journal, entries })) + + await expect(databaseMigrationStatus(database, folder)).rejects.toThrow( + 'DATABASE_MIGRATION_CATALOG_INVALID', + ) + await expect(migrateDatabase(database, folder)).rejects.toThrow( + 'DATABASE_MIGRATION_CATALOG_INVALID', + ) + expect(await exists('drizzle.__drizzle_migrations')).toBe(false) + }, + ) + + it('rejects concurrent migration ownership and succeeds after its release', async () => { + const holder = await database.sql.reserve() + try { + await holder`SELECT pg_advisory_lock(1480807217, 2)` + await expect(migrateDatabase(database)).rejects.toThrow('DATABASE_MIGRATION_BUSY') + expect(await exists('drizzle.__drizzle_migrations')).toBe(false) + } finally { + await holder`SELECT pg_advisory_unlock(1480807217, 2)` + holder.release() + } + expect(await migrateDatabase(database)).toEqual({ applied: tags, pending: [] }) + }) + + it('rolls back failed additive DDL and permits a corrected pending migration to retry', async () => { + await migrateDatabase(database) + const before = await journalRows() + const folder = await additiveFixture( + 'CREATE TABLE migration_probe (id integer);--> statement-breakpoint\nSELECT * FROM migration_missing_relation;', + ) + await expect(migrateDatabase(database, folder)).rejects.toThrow() + expect(await exists('public.migration_probe')).toBe(false) + expect(await journalRows()).toEqual(before) + const pending = await databaseMigrationStatus(database, folder) + expect(pending.applied).toEqual(tags) + expect(pending.pending).toHaveLength(1) + + await writeFile( + join(folder, `${pending.pending[0]}.sql`), + 'CREATE TABLE migration_probe (id integer);', + ) + expect(await migrateDatabase(database, folder)).toEqual({ + applied: [...tags, ...pending.pending], + pending: [], + }) + expect(await exists('public.migration_probe')).toBe(true) + }) + + it('cannot perform DDL as a restricted runtime identity', async () => { + roleName = `xcs_migration_role_${randomUUID().replaceAll('-', '')}` + if (!temporaryNames.test(roleName)) throw new Error('Invalid test role name') + await administrator.sql`CREATE ROLE ${administrator.sql(roleName)} NOLOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE` + const runtimeRole = roleName + // Apply the identity inside a real transaction; rollback also resets the role. + const restrictedSql = new Proxy(database.sql, { + get(target, property, receiver) { + if (property === 'begin') { + return async (callback: (sql: TransactionSql) => Promise) => + database.sql.begin(async (transaction) => { + await transaction`SET LOCAL ROLE ${transaction(runtimeRole)}` + return callback(transaction) + }) + } + return Reflect.get(target, property, receiver) + }, + }) + const restricted = { + ...database, + sql: restrictedSql, + } + await expect(migrateDatabase(restricted)).rejects.toMatchObject({ code: '42501' }) + expect(await exists('drizzle.__drizzle_migrations')).toBe(false) + }) +}) diff --git a/apps/indexer/test/db/provision.test.ts b/apps/indexer/test/db/provision.test.ts index 1ec875e..27fd69d 100644 --- a/apps/indexer/test/db/provision.test.ts +++ b/apps/indexer/test/db/provision.test.ts @@ -1,103 +1,87 @@ import { describe, expect, it, vi } from 'vitest' import { + bootstrapDatabase, MissingRuntimeDatabaseRolesError, - UnsafeRuntimeDatabaseRolesError, parseDatabaseClusterScope, provisionRuntimeDatabasePrivileges, + UnsafeRuntimeDatabaseRolesError, + XCS_RUNTIME_DATABASE_ROLES, } from '../../src/lib/db/bootstrap.js' import type { DatabaseClient } from '../../src/lib/db/client.js' -const ADMINISTRATOR = 'cluster_admin' - -function client(): DatabaseClient { - return { - db: {} as DatabaseClient['db'], - sql: { - begin: vi.fn(), - } as unknown as DatabaseClient['sql'], - close: vi.fn(), - } +interface FakeDatabase extends DatabaseClient { + unsafeStatements: string[] } -interface FakeRoleAttributes { - roleName: string - canLogin?: boolean - isSuperuser?: boolean - canCreateDatabase?: boolean - canCreateRole?: boolean - canReplicate?: boolean - canBypassRls?: boolean -} - -interface FakeMembership { - grantedRole: string - memberRole: string -} +const ADMINISTRATOR = 'cluster_admin' -/** A `sql` stand-in: the tagged template answers the advisory lock and the - * `current_user` lookup, and `unsafe` answers the role-attribute and - * role-membership reads and records every statement it is asked to apply. */ -function clientWithRoles( - existingRoles: readonly (string | FakeRoleAttributes)[], - memberships: readonly FakeMembership[] = [], - refuse: (statement: string) => boolean = () => false, -): DatabaseClient { - const roles = existingRoles.map((role) => (typeof role === 'string' ? { roleName: role } : role)) - const answer = (text: string): unknown[] => { - if (text.includes('rolcanlogin')) { - return roles.map((role) => ({ - roleName: role.roleName, - canLogin: role.canLogin ?? true, - isSuperuser: role.isSuperuser ?? false, - canCreateDatabase: role.canCreateDatabase ?? false, - canCreateRole: role.canCreateRole ?? false, - canReplicate: role.canReplicate ?? false, - canBypassRls: role.canBypassRls ?? false, - connectionLimit: -1, - configuration: null, - })) - } - if (text.includes('pg_auth_members')) return [...memberships] - if (text.includes('current_user')) return [{ administrator: ADMINISTRATOR }] - return [] - } - const sql = Object.assign( - (strings: TemplateStringsArray, ..._values: unknown[]) => - Promise.resolve(answer(strings.join('?'))), +function databaseWithRoles( + existingRoles: readonly string[], + options: { + unsafeAttributes?: readonly string[] + memberships?: ReadonlyArray<{ grantedRole: string; memberRole: string }> + monitorMembershipPresent?: boolean + refuse?: RegExp + } = {}, +): FakeDatabase { + const unsafeStatements: string[] = [] + const transaction = Object.assign( + vi.fn(async (strings: TemplateStringsArray) => { + const statement = strings.join('?') + if (statement.includes('current_user')) return [{ administrator: ADMINISTRATOR }] + if (statement.includes('SELECT EXISTS') && statement.includes('pg_auth_members')) { + return [{ present: options.monitorMembershipPresent ?? false }] + } + return [] + }), { - unsafe: vi.fn((text: string) => - refuse(text) - ? Promise.reject(new Error('permission denied to alter role')) - : Promise.resolve(answer(text)), - ), - savepoint: vi.fn((handler: (inner: unknown) => Promise) => handler(sql)), - begin: vi.fn((handler: (inner: unknown) => Promise) => handler(sql)), + unsafe: vi.fn(async (statement: string) => { + unsafeStatements.push(statement) + if (statement.includes('FROM pg_roles')) { + return existingRoles.map((roleName) => ({ + roleName, + canLogin: true, + isSuperuser: false, + canCreateDatabase: false, + canCreateRole: options.unsafeAttributes?.includes(roleName) ?? false, + canReplicate: false, + canBypassRls: false, + connectionLimit: -1, + configuration: [], + })) + } + if (statement.includes('FROM pg_auth_members')) return options.memberships ?? [] + if (options.refuse?.test(statement)) throw new Error('statement refused') + return [] + }), + savepoint: vi.fn(async (handler: (sql: unknown) => Promise) => handler(transaction)), }, ) return { db: {} as DatabaseClient['db'], - sql: sql as unknown as DatabaseClient['sql'], + sql: { + begin: vi.fn(async (handler: (sql: typeof transaction) => Promise) => + handler(transaction), + ), + } as unknown as DatabaseClient['sql'], close: vi.fn(), + unsafeStatements, } } -/** Only the statements provisioning applies, not the reads it answers with. */ -function appliedStatements(database: DatabaseClient): string { - return (database.sql.unsafe as unknown as { mock: { calls: [string][] } }).mock.calls - .map(([statement]) => statement) - .filter((statement) => !/^\s*SELECT/iu.test(statement)) - .join('\n') +function appliedStatements(database: FakeDatabase): string { + return database.unsafeStatements.filter((statement) => !/^\s*SELECT/iu.test(statement)).join('\n') } describe('runtime database privilege provisioning', () => { - it('requires an explicit dedicated-cluster acknowledgement', async () => { + it('requires an explicit dedicated-cluster acknowledgement before opening a transaction', async () => { expect(parseDatabaseClusterScope('dedicated')).toBe('dedicated') expect(() => parseDatabaseClusterScope(undefined)).toThrow('must be dedicated') expect(() => parseDatabaseClusterScope('shared')).toThrow('must be dedicated') - const database = client() + const database = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES) await expect( provisionRuntimeDatabasePrivileges(database, { clusterScope: 'shared' as 'dedicated', @@ -107,83 +91,46 @@ describe('runtime database privilege provisioning', () => { }) it.each([ - [[], 'xcs_indexer, xcs_api, xcs_monitor'], - [['xcs_indexer', 'xcs_monitor'], 'xcs_api'], - [['xcs_indexer', 'xcs_api'], 'xcs_monitor'], + [[], XCS_RUNTIME_DATABASE_ROLES.join(', ')], + [ + XCS_RUNTIME_DATABASE_ROLES.filter((role) => role !== 'xcs_payload_writer'), + 'xcs_payload_writer', + ], + [XCS_RUNTIME_DATABASE_ROLES.filter((role) => role !== 'xcs_issuer'), 'xcs_issuer'], ])( - 'fails by name when the managed database service has not created every role', + 'fails by name before applying SQL when DigitalOcean has not created every user', async (existingRoles, expectedNames) => { - const database = clientWithRoles(existingRoles) + const database = databaseWithRoles(existingRoles) - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).rejects.toThrow(MissingRuntimeDatabaseRolesError) - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).rejects.toThrow(expectedNames) + const operation = provisionRuntimeDatabasePrivileges(database, { + clusterScope: 'dedicated', + }) + await expect(operation).rejects.toBeInstanceOf(MissingRuntimeDatabaseRolesError) + + const repeated = provisionRuntimeDatabasePrivileges(database, { + clusterScope: 'dedicated', + }) + await expect(repeated).rejects.toThrow(expectedNames) await expect( provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), ).rejects.toThrow('doctl databases user create') - - // Nothing was applied: the run stops before any grant. expect(appliedStatements(database)).toBe('') }, ) - it.each([ - [{ roleName: 'xcs_api', canLogin: false }, 'xcs_api cannot log in'], - [{ roleName: 'xcs_api', isSuperuser: true }, 'xcs_api is a superuser'], - [{ roleName: 'xcs_api', canCreateDatabase: true }, 'xcs_api can create databases'], - [{ roleName: 'xcs_api', canCreateRole: true }, 'xcs_api can create roles'], - [{ roleName: 'xcs_api', canReplicate: true }, 'xcs_api can replicate'], - [{ roleName: 'xcs_api', canBypassRls: true }, 'xcs_api can bypass row-level security'], - ])('refuses an unsafe role attribute by name instead of altering it', async (role, expected) => { - const database = clientWithRoles(['xcs_indexer', role, 'xcs_monitor']) - - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).rejects.toBeInstanceOf(UnsafeRuntimeDatabaseRolesError) - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).rejects.toThrow(expected) - expect(appliedStatements(database)).toBe('') - }) - - it('accepts the cluster administrator as a member of every runtime role', async () => { - const database = clientWithRoles( - ['xcs_indexer', 'xcs_api', 'xcs_monitor'], - [ - { grantedRole: 'pg_monitor', memberRole: 'xcs_monitor' }, - { grantedRole: 'xcs_indexer', memberRole: ADMINISTRATOR }, - { grantedRole: 'xcs_api', memberRole: ADMINISTRATOR }, - { grantedRole: 'xcs_monitor', memberRole: ADMINISTRATOR }, - ], + it('checks every managed user before bootstrap migration DDL', async () => { + const database = databaseWithRoles( + XCS_RUNTIME_DATABASE_ROLES.filter((role) => role !== 'xcs_admin_app'), ) - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).resolves.toEqual({ administrator: ADMINISTRATOR, unappliedResourceControls: [] }) - expect(appliedStatements(database)).not.toMatch(/REVOKE\s+xcs_\w+\s+FROM/iu) - }) - - it.each([ - [{ grantedRole: 'xcs_api', memberRole: 'xcs_indexer' }, 'xcs_indexer is a member of xcs_api'], - [{ grantedRole: 'pg_monitor', memberRole: 'xcs_api' }, 'xcs_api is a member of pg_monitor'], - [ - { grantedRole: 'xcs_api', memberRole: 'someone_else' }, - 'someone_else is a member of xcs_api but is not the cluster administrator', - ], - ])('refuses a membership the managed service did not need', async (membership, expected) => { - const database = clientWithRoles(['xcs_indexer', 'xcs_api', 'xcs_monitor'], [membership]) - - await expect( - provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), - ).rejects.toThrow(expected) + await expect(bootstrapDatabase(database, { clusterScope: 'dedicated' })).rejects.toThrow( + 'xcs_admin_app', + ) expect(appliedStatements(database)).toBe('') }) - it('never creates a role, sets a password, revokes login or alters a security attribute', async () => { - const database = clientWithRoles(['xcs_indexer', 'xcs_api', 'xcs_monitor']) + it('validates and grants all eight users without managing their roles or authentication', async () => { + const database = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES) await expect( provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), @@ -194,34 +141,84 @@ describe('runtime database privilege provisioning', () => { expect(applied).not.toMatch(/PASSWORD/iu) expect(applied).not.toMatch(/NOLOGIN/iu) expect(applied).not.toMatch(/VALID UNTIL/iu) - // Role attributes are verified, never altered: these are the ones a managed - // cluster's administrator has no privilege to change. + expect(applied).not.toMatch(/RESET ALL/iu) for (const attribute of ['SUPERUSER', 'CREATEDB', 'CREATEROLE', 'REPLICATION', 'BYPASSRLS']) { expect(applied).not.toContain(attribute) } - expect(applied).not.toMatch(/RESET ALL/iu) - expect(applied).toContain('GRANT pg_monitor TO xcs_monitor') - // Resource controls are still applied, one statement each. - for (const role of ['xcs_indexer', 'xcs_api', 'xcs_monitor']) { - expect(applied).toContain(`ALTER ROLE ${role} WITH CONNECTION LIMIT`) - expect(applied).toContain(`ALTER ROLE ${role} SET statement_timeout`) - expect(applied).toContain(`ALTER ROLE ${role} SET lock_timeout`) - expect(applied).toContain(`ALTER ROLE ${role} SET idle_in_transaction_session_timeout`) + for (const role of XCS_RUNTIME_DATABASE_ROLES) { + expect(applied).toContain(role) } + expect(applied).toContain('GRANT pg_monitor TO xcs_monitor') + expect(applied).toContain('hosted_payloads, hosted_payload_publications TO xcs_payload_writer') + expect(applied).toContain( + 'app_sessions, app_auth_transactions, app_wallet_challenges TO xcs_app', + ) + expect(applied).toContain('app_admin_decisions, app_admin_notifications TO xcs_admin_app') + expect(applied).toContain('app_admin_notifications, app_admin_decisions TO xcs_notifier') + expect(applied).toContain('app_presentation_proofs TO xcs_issuer') }) - it('reports a resource control the cluster refused and that is not in effect', async () => { - const database = clientWithRoles(['xcs_indexer', 'xcs_api', 'xcs_monitor'], [], (statement) => - statement.startsWith('ALTER ROLE xcs_api SET lock_timeout'), - ) + it('fails closed on elevated attributes or unexpected memberships', async () => { + const elevated = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + unsafeAttributes: ['xcs_api'], + }) + await expect( + provisionRuntimeDatabasePrivileges(elevated, { clusterScope: 'dedicated' }), + ).rejects.toBeInstanceOf(UnsafeRuntimeDatabaseRolesError) + expect(appliedStatements(elevated)).toBe('') - const report = await provisionRuntimeDatabasePrivileges(database, { - clusterScope: 'dedicated', + const delegated = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + memberships: [{ grantedRole: 'xcs_issuer', memberRole: 'xcs_app' }], }) - expect(report.unappliedResourceControls).toEqual([ - { role: 'xcs_api', control: 'lock_timeout', intended: '15s', actual: 'unset' }, - ]) - // The grants were still applied: a missing timeout must not cost them. - expect(appliedStatements(database)).toContain('GRANT pg_monitor TO xcs_monitor') + await expect( + provisionRuntimeDatabasePrivileges(delegated, { clusterScope: 'dedicated' }), + ).rejects.toThrow('xcs_app, xcs_issuer') + expect(appliedStatements(delegated)).toBe('') + + const administered = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + memberships: XCS_RUNTIME_DATABASE_ROLES.map((grantedRole) => ({ + grantedRole, + memberRole: ADMINISTRATOR, + })), + }) + await expect( + provisionRuntimeDatabasePrivileges(administered, { clusterScope: 'dedicated' }), + ).resolves.toEqual({ administrator: ADMINISTRATOR, unappliedResourceControls: [] }) + + const monitor = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + memberships: [{ grantedRole: 'pg_monitor', memberRole: 'xcs_monitor' }], + }) + await expect( + provisionRuntimeDatabasePrivileges(monitor, { clusterScope: 'dedicated' }), + ).resolves.toEqual({ administrator: ADMINISTRATOR, unappliedResourceControls: [] }) + }) + + it('reports a refused pg_monitor grant without rolling back application grants', async () => { + const database = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + refuse: /^GRANT pg_monitor/iu, + }) + + await expect( + provisionRuntimeDatabasePrivileges(database, { clusterScope: 'dedicated' }), + ).resolves.toEqual({ + administrator: ADMINISTRATOR, + unappliedResourceControls: [ + { + role: 'xcs_monitor', + control: 'pg_monitor membership', + intended: 'granted', + actual: 'absent', + }, + ], + }) + expect(appliedStatements(database)).toContain('app_presentation_proofs TO xcs_issuer') + + const alreadyHeld = databaseWithRoles(XCS_RUNTIME_DATABASE_ROLES, { + refuse: /^GRANT pg_monitor/iu, + monitorMembershipPresent: true, + }) + await expect( + provisionRuntimeDatabasePrivileges(alreadyHeld, { clusterScope: 'dedicated' }), + ).resolves.toEqual({ administrator: ADMINISTRATOR, unappliedResourceControls: [] }) }) }) diff --git a/apps/indexer/test/db/schema.test.ts b/apps/indexer/test/db/schema.test.ts index 23521db..5fe3215 100644 --- a/apps/indexer/test/db/schema.test.ts +++ b/apps/indexer/test/db/schema.test.ts @@ -15,7 +15,7 @@ import { pinChallenges, schemaEvents, schemas, -} from '#db/schema' +} from '#db/schema/index.js' const DRIZZLE_DIRECTORY = new URL('../../../../db/migrations/', import.meta.url) @@ -111,13 +111,33 @@ describe('database schema', () => { expect(indexerStatuses.leaseExpiresAt.notNull).toBe(false) }) - it('ships one generated baseline instead of pre-production migration history', () => { + it('preserves deployed migrations and appends the application model', () => { const migrationFiles = readdirSync(DRIZZLE_DIRECTORY).filter((name) => name.endsWith('.sql')) const snapshotFiles = readdirSync(new URL('meta/', DRIZZLE_DIRECTORY)).filter((name) => name.endsWith('_snapshot.json'), ) - expect(migrationFiles).toEqual(['0000_baseline.sql']) - expect(snapshotFiles).toEqual(['0000_snapshot.json']) + expect(migrationFiles).toEqual([ + '0000_baseline.sql', + '0001_hosted_payloads.sql', + '0002_hosted_payload_locator_compatibility.sql', + '0003_application_model.sql', + '0004_auth_sessions.sql', + '0005_admin_review.sql', + '0006_issuer_workspace.sql', + '0007_recipient_verifier.sql', + '0008_presentation_wallet_proof.sql', + ]) + expect(snapshotFiles).toEqual([ + '0000_snapshot.json', + '0001_snapshot.json', + '0002_snapshot.json', + '0003_snapshot.json', + '0004_snapshot.json', + '0005_snapshot.json', + '0006_snapshot.json', + '0007_snapshot.json', + '0008_snapshot.json', + ]) const baseline = readFileSync(new URL('0000_baseline.sql', DRIZZLE_DIRECTORY), 'utf8') expect(baseline).toContain('CREATE TABLE "ledger_checkpoints"') @@ -129,6 +149,29 @@ describe('database schema', () => { const journal = JSON.parse( readFileSync(new URL('meta/_journal.json', DRIZZLE_DIRECTORY), 'utf8'), ) as { entries: Array<{ idx: number; tag: string }> } - expect(journal.entries).toEqual([expect.objectContaining({ idx: 0, tag: '0000_baseline' })]) + expect(journal.entries).toEqual([ + expect.objectContaining({ idx: 0, tag: '0000_baseline' }), + expect.objectContaining({ idx: 1, tag: '0001_hosted_payloads' }), + expect.objectContaining({ idx: 2, tag: '0002_hosted_payload_locator_compatibility' }), + expect.objectContaining({ idx: 3, tag: '0003_application_model' }), + expect.objectContaining({ idx: 4, tag: '0004_auth_sessions' }), + expect.objectContaining({ idx: 5, tag: '0005_admin_review' }), + expect.objectContaining({ idx: 6, tag: '0006_issuer_workspace' }), + expect.objectContaining({ idx: 7, tag: '0007_recipient_verifier' }), + expect.objectContaining({ idx: 8, tag: '0008_presentation_wallet_proof' }), + ]) + + const previous = JSON.parse( + readFileSync(new URL('meta/0002_snapshot.json', DRIZZLE_DIRECTORY), 'utf8'), + ) as { tables: Record } + const current = JSON.parse( + readFileSync(new URL('meta/0003_snapshot.json', DRIZZLE_DIRECTORY), 'utf8'), + ) as { tables: Record } + for (const [name, definition] of Object.entries(previous.tables)) { + expect(current.tables[name]).toEqual(definition) + } + expect(Object.keys(current.tables).filter((name) => !(name in previous.tables))).toHaveLength( + 10, + ) }) }) diff --git a/apps/indexer/test/db/serializable.test.ts b/apps/indexer/test/db/serializable.test.ts index dbc8004..e7d1d89 100644 --- a/apps/indexer/test/db/serializable.test.ts +++ b/apps/indexer/test/db/serializable.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it, vi } from 'vitest' -import type { XcsDatabase } from '../../src/lib/db/client.js' -import { runSerializableTransaction } from '../../src/lib/db/transactions.js' +import { runSerializableTransaction, type XcsDatabase } from '../../src/lib/db/index.js' function databaseWithResults(results: unknown[]) { const transaction = vi.fn( diff --git a/apps/indexer/test/generative-extraction.test.ts b/apps/indexer/test/generative-extraction.test.ts index 49715b0..7019e24 100644 --- a/apps/indexer/test/generative-extraction.test.ts +++ b/apps/indexer/test/generative-extraction.test.ts @@ -290,14 +290,14 @@ describe('generative indexer extraction', () => { }, { name: 'malformed JSON', - reasonCode: 'INVALID_JSON', + reasonCode: 'JSON_INVALID', mutate: (candidate) => { registrationMemo(candidate).MemoData = utf8Hex('{"xcsVersion":') }, }, { name: 'invalid schema', - reasonCode: 'INVALID_SCHEMA', + reasonCode: 'SCHEMA_INVALID', mutate: (candidate) => { const invalidSchema = { ...schema, future: true } as unknown as JsonValue registrationMemo(candidate).MemoData = utf8Hex(canonicalJson(invalidSchema)) diff --git a/apps/indexer/test/helpers/managedRuntimeUsers.ts b/apps/indexer/test/helpers/managedRuntimeUsers.ts new file mode 100644 index 0000000..2474b52 --- /dev/null +++ b/apps/indexer/test/helpers/managedRuntimeUsers.ts @@ -0,0 +1,70 @@ +import type { DatabaseClient } from '../../src/lib/db/client.js' + +const PASSWORD_PATTERN = /^[A-Za-z0-9_-]{32,256}$/u + +export interface ManagedRuntimeDatabasePasswords { + indexerPassword: string + apiPassword: string + payloadWriterPassword: string + monitorPassword: string + applicationPassword: string + adminApplicationPassword: string + notifierPassword: string + issuerPassword: string +} + +export const MANAGED_RUNTIME_DATABASE_PASSWORDS: ManagedRuntimeDatabasePasswords = { + indexerPassword: 'integration-only-indexer-password-0001', + apiPassword: 'integration-only-api-password-00000001', + payloadWriterPassword: 'integration-only-payload-password-001', + monitorPassword: 'integration-only-monitor-password-0001', + applicationPassword: 'integration-only-application-password-1', + adminApplicationPassword: 'integration-only-admin-password-00001', + notifierPassword: 'integration-only-notifier-password-001', + issuerPassword: 'integration-only-issuer-password-00001', +} + +const ROLE_PASSWORD_PROPERTIES = [ + ['xcs_indexer', 'indexerPassword'], + ['xcs_api', 'apiPassword'], + ['xcs_payload_writer', 'payloadWriterPassword'], + ['xcs_monitor', 'monitorPassword'], + ['xcs_app', 'applicationPassword'], + ['xcs_admin_app', 'adminApplicationPassword'], + ['xcs_notifier', 'notifierPassword'], + ['xcs_issuer', 'issuerPassword'], +] as const + +/** + * Test-only stand-in for DigitalOcean. Production bootstrap must never call this: + * the managed database service creates the users and owns their passwords. + */ +export async function createManagedRuntimeDatabaseUsers( + administrator: DatabaseClient, + overrides: Partial = {}, +): Promise { + const passwords = { ...MANAGED_RUNTIME_DATABASE_PASSWORDS, ...overrides } + for (const [, property] of ROLE_PASSWORD_PROPERTIES) { + if (!PASSWORD_PATTERN.test(passwords[property])) { + throw new Error(`Invalid test password for ${property}`) + } + } + + // Role names are fixed constants and passwords are constrained above to URL-safe + // characters. PostgreSQL utility statements do not accept password parameters. + await administrator.sql.unsafe( + ROLE_PASSWORD_PROPERTIES.map( + ([role, property]) => ` + DO $xcs_test_user$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN + CREATE ROLE ${role} LOGIN; + END IF; + END + $xcs_test_user$; + ALTER ROLE ${role} WITH LOGIN PASSWORD '${passwords[property]}'; + `, + ).join('\n'), + ) + return passwords +} diff --git a/apps/indexer/test/lib/xcs/credential.test.ts b/apps/indexer/test/lib/xcs/credential.test.ts index f524179..8d54dcf 100644 --- a/apps/indexer/test/lib/xcs/credential.test.ts +++ b/apps/indexer/test/lib/xcs/credential.test.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/test/credential.test.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/test/credential.test.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { describe, expect, it } from 'vitest' import { @@ -30,7 +30,7 @@ describe('credential payload', () => { expect(parseCredentialPayload(encoded.bytes, context)).toEqual(encoded.payload) expect(encoded.json).toBe(new TextDecoder().decode(encoded.bytes)) expect(() => parseCredentialPayload(`${encoded.json}\n`, context)).toThrow( - expect.objectContaining({ code: 'NON_CANONICAL_JSON' }), + expect.objectContaining({ code: 'PAYLOAD_INVALID' }), ) }) @@ -60,4 +60,28 @@ describe('credential payload', () => { verifyCredentialPayload({ status: 'retrieved', content: encoded.bytes }, uri, context), ).toBe('valid') }) + + it.each([{ courseId: '\ud800', passed: true }, { '\ud800': 'xcs-101' }])( + 'reports malformed Unicode in payload values or keys as invalid', + (claims) => { + const content = JSON.stringify({ + claims, + issuer: context.issuer, + schema: context.schemaUid, + subject: context.subject, + xcsVersion: '0.1', + }) + const uri = createIpfsPayloadUri(content) + + expect(verifyCredentialPayload({ status: 'retrieved', content }, uri, context)).toBe( + 'invalid', + ) + }, + ) + + it('preserves valid Unicode including surrogate pairs', () => { + const encoded = encodeCredentialPayload({ courseId: 'Cours réussi 🎓', passed: true }, context) + + expect(parseCredentialPayload(encoded.bytes, context).claims.courseId).toBe('Cours réussi 🎓') + }) }) diff --git a/apps/indexer/test/lib/xcs/network-lifecycle.test.ts b/apps/indexer/test/lib/xcs/network-lifecycle.test.ts index b75ba94..8888e31 100644 --- a/apps/indexer/test/lib/xcs/network-lifecycle.test.ts +++ b/apps/indexer/test/lib/xcs/network-lifecycle.test.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/test/network-lifecycle.test.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/test/network-lifecycle.test.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { describe, expect, it } from 'vitest' import { @@ -27,7 +27,7 @@ describe('network and native lifecycle', () => { it('wraps xrpl.js Ripple-time conversion with strict protocol bounds', () => { expect(rippleTimeToIso(0)).toBe('2000-01-01T00:00:00.000Z') expect(() => rippleTimeToIso(-1)).toThrow( - expect.objectContaining({ code: 'INVALID_RIPPLE_TIME' }), + expect.objectContaining({ code: 'RIPPLE_TIME_INVALID' }), ) }) diff --git a/apps/indexer/test/lib/xcs/schema.test.ts b/apps/indexer/test/lib/xcs/schema.test.ts index 333b15f..b0acef2 100644 --- a/apps/indexer/test/lib/xcs/schema.test.ts +++ b/apps/indexer/test/lib/xcs/schema.test.ts @@ -1,4 +1,4 @@ -// Copied from packages/core/test/schema.test.ts at 54c3486; keep in sync by hand (see CONTRIBUTING.md). +// Copied from packages/core/test/schema.test.ts at a9777cc; keep in sync by hand (see CONTRIBUTING.md). import { describe, expect, it } from 'vitest' import { @@ -45,7 +45,7 @@ describe('schema', () => { ) expect(parseSchemaBytes(canonical).name).toBe('Schema') expect(() => parseSchemaBytes(new TextEncoder().encode('{ "xcsVersion": "0.1" }'))).toThrow( - expect.objectContaining({ code: 'NON_CANONICAL_JSON' }), + expect.objectContaining({ code: 'JSON_INVALID' }), ) expect(() => parseSchemaBytes( @@ -53,7 +53,7 @@ describe('schema', () => { '{"description":"First","description":"Second","fields":{"value":{"type":"string"}},"name":"Schema","xcsVersion":"0.1"}', ), ), - ).toThrow(expect.objectContaining({ code: 'INVALID_JSON' })) + ).toThrow(expect.objectContaining({ code: 'JSON_DUPLICATE_KEY' })) }) it('resolves prior parents and rejects field overrides', () => { @@ -89,7 +89,7 @@ describe('schema', () => { expect(Object.keys(resolveSchema(child, context).fields)).toEqual(['courseId', 'score']) expect(() => resolveSchema(parseSchema({ ...child, fields: { courseId: { type: 'bool' } } }), context), - ).toThrow(expect.objectContaining({ code: 'INVALID_SCHEMA_REFERENCE' })) + ).toThrow(expect.objectContaining({ code: 'SCHEMA_OVERRIDE_FORBIDDEN' })) }) it('derives the published schema UID with noble SHA-256', () => { diff --git a/apps/indexer/test/postgres.integration.test.ts b/apps/indexer/test/postgres.integration.test.ts index a5b6805..ccb1179 100644 --- a/apps/indexer/test/postgres.integration.test.ts +++ b/apps/indexer/test/postgres.integration.test.ts @@ -1,40 +1,41 @@ -import { randomUUID } from 'node:crypto' -import { mkdtemp, rm } from 'node:fs/promises' +import { createHash, randomUUID } from 'node:crypto' +import { copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' +import { fileURLToPath } from 'node:url' import { + acquireIndexerLease, credentialEvents, credentialGenerations, + createDatabaseClient, + haltIndexer, indexerIncidents, indexerStatuses, ledgerCheckpoints, - schemaEvents, - schemas, -} from '#db/schema' -import { type DatabaseClient, createDatabaseClient } from '../src/lib/db/client.js' -import { - acquireIndexerLease, - haltIndexer, releaseIndexerLease, renewIndexerLease, + schemaEvents, + schemas, updateIndexerStatus, -} from '../src/lib/db/indexer-fencing.js' + type DatabaseClient, +} from '../src/lib/db/index.js' import { + initializeDatabase, + provisionRuntimeDatabasePrivileges, XCS_API_DATABASE_CONNECTION_LIMIT, XCS_INDEXER_DATABASE_CONNECTION_LIMIT, XCS_MONITOR_DATABASE_CONNECTION_LIMIT, - MissingRuntimeDatabaseRolesError, - UnsafeRuntimeDatabaseRolesError, - initializeDatabase, - provisionRuntimeDatabasePrivileges, + XCS_RUNTIME_DATABASE_ROLES, } from '../src/lib/db/bootstrap.js' import { computeSchemaUid, createIpfsPayloadUri, type JsonValue } from '../src/lib/xcs/index.js' import { and, asc, eq } from 'drizzle-orm' +import { migrate } from 'drizzle-orm/postgres-js/migrator' import { afterAll, beforeAll, describe, expect, it } from 'vitest' import { captureLedgerFixtureBundle, ledgerFixtureBundleDigest } from '../src/fixture-bundle.js' import { prepareFixtureReplay } from '../src/fixture-replay.js' +import { createManagedRuntimeDatabaseUsers } from './helpers/managedRuntimeUsers.js' import { computeProjectionDigest } from '../src/projection-digest.js' import { QuorumLedgerSource } from '../src/quorum-ledger-source.js' import { PostgresIndexerRepository } from '../src/repository.js' @@ -112,6 +113,7 @@ let runtimeRoleCleanupAllowed = false const INDEXER_DATABASE_PASSWORD = 'indexer-integration-password-000001' const API_DATABASE_PASSWORD = 'api-integration-password-0000000001' +const PAYLOAD_DATABASE_PASSWORD = 'payload-integration-password-00000001' const MONITOR_DATABASE_PASSWORD = 'monitor-integration-password-00000001' function temporaryDatabaseName(): string { @@ -164,7 +166,12 @@ async function closeAndDropTemporaryDatabases(): Promise { DROP ROLE IF EXISTS xcs_indexer, xcs_api, - xcs_monitor + xcs_payload_writer, + xcs_monitor, + xcs_app, + xcs_admin_app, + xcs_notifier, + xcs_issuer ` runtimeRoleCleanupAllowed = false } catch (error) { @@ -824,6 +831,58 @@ describePostgres('PostgreSQL 18 indexer integration', () => { expect(integrityConstraints.every((constraint) => constraint.validated)).toBe(true) }) + it('upgrades the baseline and legacy hosted storage without losing profiles or payloads', async () => { + if (adminDatabaseUrl === undefined) throw new Error('PostgreSQL admin URL is not initialized') + const database = await createTemporaryDatabase(adminDatabaseUrl, { initialize: false }) + const source = fileURLToPath(new URL('../../../db/migrations/', import.meta.url)) + const fixture = await mkdtemp(join(tmpdir(), 'xcs-migration-upgrade-')) + const journal = JSON.parse(await readFile(join(source, 'meta/_journal.json'), 'utf8')) + const content = '{"course":"test-upgrade"}' + const digest = createHash('sha256').update(content).digest('hex') + try { + await mkdir(join(fixture, 'meta')) + for (const count of [1, 2]) { + const entries = journal.entries.slice(0, count) + for (const entry of entries) { + await copyFile(join(source, `${entry.tag}.sql`), join(fixture, `${entry.tag}.sql`)) + } + await writeFile( + join(fixture, 'meta/_journal.json'), + JSON.stringify({ ...journal, entries }), + ) + await migrate(database.client.db, { migrationsFolder: fixture }) + if (count === 1) { + await new PostgresIndexerRepository(database.client.db).initializeProfile( + profile('upgrade'), + ) + } else { + await database.client.sql` + INSERT INTO hosted_payloads (locator, digest_hex, content) + VALUES (${digest.slice(0, 20)}, ${digest}, ${content}) + ` + } + } + await initializeDatabase(database.client) + await initializeDatabase(database.client) + const profiles = await database.client.sql`SELECT profile_id FROM network_profiles` + expect(profiles.map((row) => row.profile_id)).toEqual(['upgrade']) + // The same bytes may have an old 20-character link and a new 18-character link. + await database.client.sql` + INSERT INTO hosted_payloads (locator, digest_hex, content) + VALUES (${digest.slice(0, 18)}, ${digest}, ${content}) + ` + const payloads = await database.client.sql` + SELECT locator, content FROM hosted_payloads ORDER BY length(locator) + ` + expect(payloads).toEqual([ + { locator: digest.slice(0, 18), content }, + { locator: digest.slice(0, 20), content }, + ]) + } finally { + await rm(fixture, { recursive: true, force: true }) + } + }) + it('allows exact restarts but rejects another profile in exclusive database scope', async () => { if (adminDatabaseUrl === undefined) throw new Error('PostgreSQL admin URL is not initialized') const database = await createTemporaryDatabase(adminDatabaseUrl) @@ -861,67 +920,37 @@ describePostgres('PostgreSQL 18 indexer integration', () => { }) }) - it('provisions idempotent least-privilege indexer, API and monitor roles', async () => { + it('provisions idempotent least-privilege indexer, read API, payload writer and monitor roles', async () => { const database = temporaryDatabases[0] if (database === undefined) throw new Error('First temporary database was not created') if (adminClient === undefined) throw new Error('PostgreSQL admin client is not initialized') const provisioning = { clusterScope: 'dedicated' } as const - // The roles do not exist yet: provisioning is grants-only and must say so by name. - await expect( - provisionRuntimeDatabasePrivileges(database.client, provisioning), - ).rejects.toBeInstanceOf(MissingRuntimeDatabaseRolesError) - await expect(provisionRuntimeDatabasePrivileges(database.client, provisioning)).rejects.toThrow( - 'xcs_indexer, xcs_api, xcs_monitor', - ) - - // Stands in for the managed database service: it, not this step, owns the - // users and their passwords. runtimeRoleCleanupAllowed = true - await adminClient.sql.unsafe(` - CREATE ROLE xcs_indexer LOGIN PASSWORD '${INDEXER_DATABASE_PASSWORD}'; - CREATE ROLE xcs_api LOGIN PASSWORD '${API_DATABASE_PASSWORD}'; - CREATE ROLE xcs_monitor LOGIN PASSWORD '${MONITOR_DATABASE_PASSWORD}'; - GRANT xcs_indexer, xcs_api, xcs_monitor TO CURRENT_USER WITH ADMIN OPTION; - `) - - // Reproduces a managed cluster: the platform makes its administrator a - // member of every user it creates. That membership is how the service - // administers those users, so provisioning accepts it instead of trying to - // revoke it -- which the administrator cannot do, not being the grantor. - const firstRun = await provisionRuntimeDatabasePrivileges(database.client, provisioning) - const secondRun = await provisionRuntimeDatabasePrivileges(database.client, provisioning) - const [session] = await adminClient.sql<{ administrator: string }[]>` - SELECT current_user AS "administrator" + await createManagedRuntimeDatabaseUsers(adminClient, { + indexerPassword: INDEXER_DATABASE_PASSWORD, + apiPassword: API_DATABASE_PASSWORD, + payloadWriterPassword: PAYLOAD_DATABASE_PASSWORD, + monitorPassword: MONITOR_DATABASE_PASSWORD, + }) + const firstReport = await provisionRuntimeDatabasePrivileges(database.client, provisioning) + expect(firstReport).toEqual({ + administrator: expect.any(String), + unappliedResourceControls: [], + }) + // Simulate the pre-split API grants: reprovisioning must remove them, not just add a writer. + await database.client.sql` + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE pin_challenges, demo_pins TO xcs_api + ` + await database.client.sql` + GRANT SELECT, INSERT ON TABLE hosted_payloads, hosted_payload_publications TO xcs_api ` - expect(firstRun).toEqual({ - administrator: session?.administrator, + const repeatedReport = await provisionRuntimeDatabasePrivileges(database.client, provisioning) + expect(repeatedReport).toEqual({ + administrator: firstReport.administrator, unappliedResourceControls: [], }) - expect(secondRun).toEqual(firstRun) - - // An unsafe attribute is verified, not altered away: provisioning cannot - // change it without a superuser, so it refuses and names the role. - await adminClient.sql.unsafe('ALTER ROLE xcs_api WITH CREATEDB') - await expect( - provisionRuntimeDatabasePrivileges(database.client, provisioning), - ).rejects.toBeInstanceOf(UnsafeRuntimeDatabaseRolesError) - await expect(provisionRuntimeDatabasePrivileges(database.client, provisioning)).rejects.toThrow( - 'xcs_api can create databases', - ) - await adminClient.sql.unsafe('ALTER ROLE xcs_api WITH NOCREATEDB') - - // A membership between two runtime roles, or held by anyone other than the - // cluster administrator, is still rejected. - await adminClient.sql.unsafe('GRANT xcs_api TO xcs_indexer') - await expect(provisionRuntimeDatabasePrivileges(database.client, provisioning)).rejects.toThrow( - 'xcs_indexer is a member of xcs_api', - ) - await adminClient.sql.unsafe('REVOKE xcs_api FROM xcs_indexer') - - // And the cluster is back to a state provisioning accepts. - await provisionRuntimeDatabasePrivileges(database.client, provisioning) const roleProperties = await adminClient.sql< Array<{ @@ -953,46 +982,19 @@ describePostgres('PostgreSQL 18 indexer integration', () => { ORDER BY setting ) AS configuration FROM pg_roles - WHERE rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + WHERE rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) ORDER BY rolname ` - expect(roleProperties).toEqual([ - { - roleName: 'xcs_api', - canLogin: true, - isSuperuser: false, - canCreateDatabase: false, - canCreateRole: false, - canReplicate: false, - canBypassRls: false, - // Left as the managed service created it: INHERIT is not in the unsafe - // set, and with no membership beyond pg_monitor -> xcs_monitor it is inert. - inheritsPrivileges: true, - connectionLimit: XCS_API_DATABASE_CONNECTION_LIMIT, - configuration: [ - 'idle_in_transaction_session_timeout=30s', - 'lock_timeout=15s', - 'statement_timeout=30s', - ], - }, - { - roleName: 'xcs_indexer', - canLogin: true, - isSuperuser: false, - canCreateDatabase: false, - canCreateRole: false, - canReplicate: false, - canBypassRls: false, - inheritsPrivileges: true, - connectionLimit: XCS_INDEXER_DATABASE_CONNECTION_LIMIT, - configuration: [ - 'idle_in_transaction_session_timeout=30s', - 'lock_timeout=30s', - 'statement_timeout=5min', - ], - }, - { - roleName: 'xcs_monitor', + expect(roleProperties.map(({ roleName }) => roleName)).toEqual( + [...XCS_RUNTIME_DATABASE_ROLES].sort(), + ) + for (const role of roleProperties) { + const monitor = role.roleName === 'xcs_monitor' + const indexer = role.roleName === 'xcs_indexer' + expect(role).toMatchObject({ canLogin: true, isSuperuser: false, canCreateDatabase: false, @@ -1000,14 +1002,18 @@ describePostgres('PostgreSQL 18 indexer integration', () => { canReplicate: false, canBypassRls: false, inheritsPrivileges: true, - connectionLimit: XCS_MONITOR_DATABASE_CONNECTION_LIMIT, + connectionLimit: monitor + ? XCS_MONITOR_DATABASE_CONNECTION_LIMIT + : indexer + ? XCS_INDEXER_DATABASE_CONNECTION_LIMIT + : XCS_API_DATABASE_CONNECTION_LIMIT, configuration: [ 'idle_in_transaction_session_timeout=30s', - 'lock_timeout=10s', - 'statement_timeout=30s', + `lock_timeout=${monitor ? '10s' : indexer ? '30s' : '15s'}`, + `statement_timeout=${indexer ? '5min' : '30s'}`, ], - }, - ]) + }) + } const runtimeMemberships = await adminClient.sql< Array<{ @@ -1027,7 +1033,10 @@ describePostgres('PostgreSQL 18 indexer integration', () => { FROM pg_auth_members membership JOIN pg_roles granted_role ON granted_role.oid = membership.roleid JOIN pg_roles member_role ON member_role.oid = membership.member - WHERE member_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + WHERE member_role.rolname IN ( + 'xcs_indexer', 'xcs_api', 'xcs_payload_writer', 'xcs_monitor', + 'xcs_app', 'xcs_admin_app', 'xcs_notifier', 'xcs_issuer' + ) ORDER BY granted_role.rolname, member_role.rolname ` expect(runtimeMemberships).toEqual([ @@ -1061,173 +1070,48 @@ describePostgres('PostgreSQL 18 indexer integration', () => { JOIN pg_roles grantee_role ON grantee_role.oid = privilege.grantee WHERE namespace_object.nspname = 'public' AND relation.relkind IN ('r', 'p') - AND grantee_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + AND grantee_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor', 'xcs_payload_writer') ORDER BY grantee_role.rolname, relation.relname, privilege.privilege_type ` - expect(tableGrants).toEqual([ - { - grantee: 'xcs_api', - tableName: 'credential_events', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'credential_generations', - privilegeType: 'SELECT', - grantable: false, - }, - { grantee: 'xcs_api', tableName: 'demo_pins', privilegeType: 'DELETE', grantable: false }, - { grantee: 'xcs_api', tableName: 'demo_pins', privilegeType: 'INSERT', grantable: false }, - { grantee: 'xcs_api', tableName: 'demo_pins', privilegeType: 'SELECT', grantable: false }, - { grantee: 'xcs_api', tableName: 'demo_pins', privilegeType: 'UPDATE', grantable: false }, - { - grantee: 'xcs_api', - tableName: 'indexer_incidents', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'indexer_status', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'ledger_checkpoints', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'network_profiles', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'pin_challenges', - privilegeType: 'DELETE', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'pin_challenges', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'pin_challenges', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_api', - tableName: 'pin_challenges', - privilegeType: 'UPDATE', - grantable: false, - }, - { grantee: 'xcs_api', tableName: 'schema_events', privilegeType: 'SELECT', grantable: false }, - { grantee: 'xcs_api', tableName: 'schemas', privilegeType: 'SELECT', grantable: false }, - { - grantee: 'xcs_indexer', - tableName: 'credential_events', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'credential_events', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'credential_generations', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'credential_generations', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'indexer_incidents', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'indexer_incidents', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'indexer_status', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'indexer_status', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'ledger_checkpoints', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'ledger_checkpoints', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'network_profiles', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'network_profiles', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'schema_events', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'schema_events', - privilegeType: 'SELECT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'schemas', - privilegeType: 'INSERT', - grantable: false, - }, - { - grantee: 'xcs_indexer', - tableName: 'schemas', - privilegeType: 'SELECT', - grantable: false, - }, - ]) + const projectionTables = [ + 'credential_events', + 'credential_generations', + 'indexer_incidents', + 'indexer_status', + 'ledger_checkpoints', + 'network_profiles', + 'schema_events', + 'schemas', + ] + const expectedTableGrants = [ + ...projectionTables.flatMap((tableName) => [ + { grantee: 'xcs_api', tableName, privilegeType: 'SELECT', grantable: false }, + ...['SELECT', 'INSERT'].map((privilegeType) => ({ + grantee: 'xcs_indexer', + tableName, + privilegeType, + grantable: false, + })), + ]), + ...['demo_pins', 'pin_challenges'].flatMap((tableName) => + ['SELECT', 'INSERT', 'UPDATE', 'DELETE'].map((privilegeType) => ({ + grantee: 'xcs_payload_writer', + tableName, + privilegeType, + grantable: false, + })), + ), + ...['hosted_payloads', 'hosted_payload_publications'].flatMap((tableName) => + ['SELECT', 'INSERT'].map((privilegeType) => ({ + grantee: 'xcs_payload_writer', + tableName, + privilegeType, + grantable: false, + })), + ), + ] + expect(tableGrants).toHaveLength(expectedTableGrants.length) + expect(tableGrants).toEqual(expect.arrayContaining(expectedTableGrants)) const columnGrants = await database.client.sql< Array<{ @@ -1252,7 +1136,7 @@ describePostgres('PostgreSQL 18 indexer integration', () => { WHERE namespace_object.nspname = 'public' AND attribute_object.attnum > 0 AND NOT attribute_object.attisdropped - AND grantee_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor') + AND grantee_role.rolname IN ('xcs_indexer', 'xcs_api', 'xcs_monitor', 'xcs_payload_writer') ORDER BY grantee_role.rolname, relation.relname, attribute_object.attname, privilege.privilege_type ` expect(columnGrants).toEqual([ @@ -1369,6 +1253,9 @@ describePostgres('PostgreSQL 18 indexer integration', () => { const apiClient = createDatabaseClient( runtimeDatabaseUrl(database.url, 'xcs_api', API_DATABASE_PASSWORD), ) + const payloadClient = createDatabaseClient( + runtimeDatabaseUrl(database.url, 'xcs_payload_writer', PAYLOAD_DATABASE_PASSWORD), + ) const monitorClient = createDatabaseClient( runtimeDatabaseUrl(database.url, 'xcs_monitor', MONITOR_DATABASE_PASSWORD), ) @@ -1554,7 +1441,7 @@ describePostgres('PostgreSQL 18 indexer integration', () => { const challengeId = '1'.repeat(64) const pinId = '2'.repeat(64) - await apiClient.sql` + await payloadClient.sql` INSERT INTO pin_challenges ( challenge_id, profile_id, wallet, requester_ip_hash, message, expires_at ) VALUES ( @@ -1562,7 +1449,7 @@ describePostgres('PostgreSQL 18 indexer integration', () => { ${'3'.repeat(64)}, 'authorized', CURRENT_TIMESTAMP + interval '5 minutes' ) ` - await apiClient.sql` + await payloadClient.sql` INSERT INTO demo_pins ( pin_id, challenge_id, profile_id, wallet, requester_ip_hash, cid, byte_length, status, expires_at @@ -1572,20 +1459,58 @@ describePostgres('PostgreSQL 18 indexer integration', () => { CURRENT_TIMESTAMP + interval '1 hour' ) ` - const [pinRead] = await apiClient.sql<{ status: string }[]>` + const [pinRead] = await payloadClient.sql<{ status: string }[]>` SELECT status FROM demo_pins WHERE pin_id = ${pinId} ` expect(pinRead?.status).toBe('pending') - await apiClient.sql` + await payloadClient.sql` UPDATE demo_pins SET status = 'pinned' WHERE pin_id = ${pinId} ` - await apiClient.sql`DELETE FROM demo_pins WHERE pin_id = ${pinId}` - await apiClient.sql`DELETE FROM pin_challenges WHERE challenge_id = ${challengeId}` + await payloadClient.sql`DELETE FROM demo_pins WHERE pin_id = ${pinId}` + await payloadClient.sql`DELETE FROM pin_challenges WHERE challenge_id = ${challengeId}` + for (const runtimeClient of [apiClient, payloadClient]) { + for (const tableName of projectionTables) { + await expectPermissionDenied( + runtimeClient.sql`DELETE FROM ${runtimeClient.sql(tableName)} WHERE false`, + ) + await expectPermissionDenied( + runtimeClient.sql`INSERT INTO ${runtimeClient.sql(tableName)} DEFAULT VALUES`, + ) + } + await expectPermissionDenied(runtimeClient.sql`SET ROLE xcs_indexer`) + await expectPermissionDenied(runtimeClient.sql`SET ROLE xcs_monitor`) + } + await expectPermissionDenied(apiClient.sql`SET ROLE xcs_payload_writer`) + await expectPermissionDenied(payloadClient.sql`SET ROLE xcs_api`) + await expectPermissionDenied(payloadClient.sql`SELECT * FROM schemas`) + await expectPermissionDenied( + payloadClient.sql`UPDATE credential_generations SET accepted = true WHERE false`, + ) + for (const tableName of [ + 'demo_pins', + 'pin_challenges', + 'hosted_payloads', + 'hosted_payload_publications', + ]) { + await expectPermissionDenied(apiClient.sql`SELECT * FROM ${apiClient.sql(tableName)}`) + await expectPermissionDenied( + apiClient.sql`INSERT INTO ${apiClient.sql(tableName)} DEFAULT VALUES`, + ) + await expectPermissionDenied( + apiClient.sql`DELETE FROM ${apiClient.sql(tableName)} WHERE false`, + ) + } + await expectPermissionDenied(payloadClient.sql`CREATE TABLE forbidden_payload (id integer)`) await expectPermissionDenied(apiClient.sql`CREATE TABLE forbidden_api (id integer)`) await expectPermissionDenied(indexerClient.sql`CREATE TABLE forbidden_indexer (id integer)`) } finally { - await Promise.allSettled([indexerClient.close(), apiClient.close(), monitorClient.close()]) + await Promise.allSettled([ + indexerClient.close(), + apiClient.close(), + payloadClient.close(), + monitorClient.close(), + ]) } }, 120_000) diff --git a/apps/indexer/test/registration.test.ts b/apps/indexer/test/registration.test.ts index 200309c..3ef0293 100644 --- a/apps/indexer/test/registration.test.ts +++ b/apps/indexer/test/registration.test.ts @@ -256,6 +256,6 @@ describe('interpretSchemaRegistration', () => { profile, new Map(), ), - ).toMatchObject({ status: 'rejected', reasonCode: 'INVALID_JSON' }) + ).toMatchObject({ status: 'rejected', reasonCode: 'JSON_INVALID' }) }) }) diff --git a/apps/indexer/test/xrpl-source.test.ts b/apps/indexer/test/xrpl-source.test.ts index 1a3e18e..b378519 100644 --- a/apps/indexer/test/xrpl-source.test.ts +++ b/apps/indexer/test/xrpl-source.test.ts @@ -1,8 +1,11 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' +import { encode } from 'ripple-binary-codec' +import { Client, decode, hashes, Wallet } from 'xrpl' import { assertFeatureResponseSupportsAmendment, normalizeLedgerResponse, + XrplLedgerSource, } from '../src/xrpl-source.js' const LEDGER_HASH = 'a'.repeat(64) @@ -102,6 +105,41 @@ describe('normalizeLedgerResponse', () => { }) }) + it.each<[string, unknown]>([ + ['nftoken_id', FIRST_TX_HASH], + ['nftoken_ids', [FIRST_TX_HASH]], + ['offer_id', FIRST_TX_HASH], + ['mpt_issuance_id', '1'.repeat(48)], + ['delivered_amount', '1'], + ])( + 'ignores the synthetic API metadata field %s without mutating the response', + (field, value) => { + const decorated = response() + const metadata = record(transactions(decorated)[0]!.meta) + metadata[field] = value + + expect(normalizeLedgerResponse(decorated)).toEqual(normalizeLedgerResponse(response())) + expect(metadata[field]).toEqual(value) + }, + ) + + it.each<[string, unknown]>([ + ['DeliveredAmount', '1'], + ['AffectedNodes', [{ ModifiedNode: { LedgerEntryType: 'AccountRoot' } }]], + ['TransactionResult', 'tecFAILED'], + ['FutureCanonicalField', 'preserved'], + ])('preserves %s for the quorum to detect canonical metadata differences', (field, value) => { + const changed = response() + const metadata = record(transactions(changed)[0]!.meta) + metadata[field] = value + metadata.delivered_amount = '1' + + const normalized = normalizeLedgerResponse(changed) + expect(normalized).not.toEqual(normalizeLedgerResponse(response())) + expect(normalized.transactions[1]!.metadata[field]).toEqual(value) + expect(normalized.transactions[1]!.metadata).not.toHaveProperty('delivered_amount') + }) + it('rejects a ledger that is not marked validated or closed', () => { const unvalidated = response() unvalidated.validated = false @@ -242,3 +280,173 @@ describe('assertFeatureResponseSupportsAmendment', () => { ) }) }) + +// Published XRPL binary-codec header vector (ledger 32052277); no transaction/account secrets. +const BINARY_HEADER = + '01E91435016340767BF1C4A3EACEB081770D8ADE216C85445DD6FB002C6B5A2930F2DECE006DA18150CB18F6DD33F6F0990754C962A7CCE62F332FF9C13939B03B864117F0BDA86B6E9B4F873B5C3E520634D343EF5D9D9A4246643D64DAD278BA95DC0EAC6EB5350CF970D521276CDE21276CE60A00' +const BINARY_HEADER_FIELDS = { + ledger_index: 32052277, + total_coins: '99994494362043555', + parent_hash: 'EACEB081770D8ADE216C85445DD6FB002C6B5A2930F2DECE006DA18150CB18F6', + transaction_hash: 'DD33F6F0990754C962A7CCE62F332FF9C13939B03B864117F0BDA86B6E9B4F87', + account_hash: '3B5C3E520634D343EF5D9D9A4246643D64DAD278BA95DC0EAC6EB5350CF970D5', + parent_close_time: 556231902, + close_time: 556231910, + close_time_resolution: 10, + close_flags: 0, +} +function binaryResponse() { + const holder = Wallet.generate() + const tx = holder.sign({ + TransactionType: 'Payment', + Account: holder.address, + Destination: Wallet.generate().address, + Amount: '123', + Sequence: 1, + Fee: '12', + }) + const metadata = { + TransactionIndex: 0, + TransactionResult: 'tesSUCCESS', + AffectedNodes: [], + DeliveredAmount: '123', + } + return { + signed: tx, + metadata, + result: { + validated: true, + ledger_index: BINARY_HEADER_FIELDS.ledger_index, + ledger_hash: hashes.hashLedgerHeader( + BINARY_HEADER_FIELDS as Parameters[0], + ), + ledger: { + closed: true, + ledger_data: BINARY_HEADER, + transactions: [{ tx_blob: tx.tx_blob, meta: encode(metadata) }], + }, + }, + } +} + +describe('binary ledger transport', () => { + it('decodes canonical header/transaction/metadata and derives the missing transaction hash from signed bytes', () => { + const fixture = binaryResponse() + const normalized = normalizeLedgerResponse(fixture.result) + expect(normalized).toMatchObject({ + ledgerIndex: 32052277, + totalCoins: '99994494362043555', + parentCloseTime: 556231902, + closeTime: 556231910, + closeTimeResolution: 10, + closeFlags: 0, + parentHash: BINARY_HEADER_FIELDS.parent_hash.toLowerCase(), + transactionRoot: BINARY_HEADER_FIELDS.transaction_hash.toLowerCase(), + accountRoot: BINARY_HEADER_FIELDS.account_hash.toLowerCase(), + }) + expect(normalized.transactions[0]).toMatchObject({ + hash: fixture.signed.hash.toLowerCase(), + transactionIndex: 0, + metadata: fixture.metadata, + transaction: { TransactionType: 'Payment', Amount: '123' }, + }) + expect(normalized.transactions[0]!.transaction).not.toHaveProperty('DeliverMax') + const expanded = { + ...fixture.result, + ledger: { + ...BINARY_HEADER_FIELDS, + ledger_hash: fixture.result.ledger_hash, + closed: true, + transactions: [ + { + hash: fixture.signed.hash, + tx_json: decode(fixture.signed.tx_blob), + meta: fixture.metadata, + }, + ], + }, + } + expect(normalized).toEqual(normalizeLedgerResponse(expanded)) + }) + it('accepts API v2 meta_blob and hash while rejecting conflicting metadata aliases', () => { + const fixture = binaryResponse() + const expected = normalizeLedgerResponse(fixture.result) + const tx = record(fixture.result.ledger.transactions[0]) + tx.meta_blob = tx.meta + tx.hash = fixture.signed.hash + delete tx.meta + expect(normalizeLedgerResponse(fixture.result)).toEqual(expected) + tx.meta = encode({ ...fixture.metadata, TransactionResult: 'tecUNFUNDED_PAYMENT' }) + expect(() => normalizeLedgerResponse(fixture.result)).toThrow( + 'metadata representations disagree', + ) + }) + it('requests a complete binary ledger and keeps the response-index guard', async () => { + const fixture = binaryResponse() + const request = vi + .spyOn(Client.prototype, 'request') + .mockResolvedValue({ result: fixture.result } as never) + try { + const source = new XrplLedgerSource('wss://source.invalid') + expect(await source.getLedger(32052277)).toEqual(normalizeLedgerResponse(fixture.result)) + expect(request).toHaveBeenCalledWith({ + command: 'ledger', + ledger_index: 32052277, + transactions: true, + expand: true, + binary: true, + }) + await expect(source.getLedger(32052278)).rejects.toMatchObject({ + code: 'SOURCE_RESPONSE_INVALID', + }) + } finally { + request.mockRestore() + } + }) + it.each(['', '0', 'GG', '00', BINARY_HEADER + '00'])( + 'rejects malformed or trailing binary header bytes %s', + (header) => { + const fixture = binaryResponse() + fixture.result.ledger.ledger_data = header + expect(() => normalizeLedgerResponse(fixture.result)).toThrow() + }, + ) + it('rejects a binary header that disagrees with the advertised hash', () => { + const fixture = binaryResponse() + fixture.result.ledger_hash = '0'.repeat(64) + expect(() => normalizeLedgerResponse(fixture.result)).toThrow('hash values disagree') + }) + it.each(['tx_blob', 'meta'] as const)( + 'rejects invalid binary %s instead of trusting optional JSON shadows', + (field) => { + const fixture = binaryResponse() + const tx = fixture.result.ledger.transactions[0]! + tx[field] = '00' + Object.assign(tx, { hash: fixture.signed.hash, tx_json: decode(fixture.signed.tx_blob) }) + expect(() => normalizeLedgerResponse(fixture.result)).toThrow('Cannot decode binary') + }, + ) + it('rejects a conflicting optional transaction hash', () => { + const fixture = binaryResponse() + Object.assign(fixture.result.ledger.transactions[0]!, { hash: '0'.repeat(64) }) + expect(() => normalizeLedgerResponse(fixture.result)).toThrow('hash values disagree') + }) + it('retains legitimate protocol pseudo-transactions without claiming an account signature', () => { + const fixture = binaryResponse() + const pseudo = { + TransactionType: 'EnableAmendment', + Account: 'rrrrrrrrrrrrrrrrrrrrrhoLvTp', + LedgerSequence: 32052277, + Amendment: 'a'.repeat(64), + } + const blob = encode(pseudo) + fixture.result.ledger.transactions[0]!.tx_blob = blob + const normalized = normalizeLedgerResponse(fixture.result).transactions[0]! + expect(normalized.transaction).toMatchObject({ + ...pseudo, + Amendment: pseudo.Amendment.toUpperCase(), + }) + // The standard ID domain applies to unsigned protocol operations too. + expect(normalized.hash).toBe('38b05da5a7dc38038355bb22ce9b605ba616f1cf87e8576dd6be3da1a6bf1702') + }) +}) diff --git a/apps/indexer/tsup.config.ts b/apps/indexer/tsup.config.ts index c072052..a4de7ba 100644 --- a/apps/indexer/tsup.config.ts +++ b/apps/indexer/tsup.config.ts @@ -10,6 +10,8 @@ export default defineConfig({ 'src/fixture-cli.ts', 'src/lib/db/bin/bootstrap.ts', 'src/lib/db/bin/migrate.ts', + 'src/lib/db/bin/status.ts', + 'src/lib/db/bin/admin-bootstrap.ts', ], format: ['esm'], dts: false, diff --git a/apps/indexer/vitest.config.ts b/apps/indexer/vitest.config.ts index 0ce4b3b..4d5f7f0 100644 --- a/apps/indexer/vitest.config.ts +++ b/apps/indexer/vitest.config.ts @@ -9,5 +9,7 @@ export default defineConfig({ test: { environment: 'node', include: ['test/**/*.test.ts'], + // Integration files provision the same cluster-wide roles, even in separate databases. + fileParallelism: !process.env.XCS_TEST_DATABASE_URL?.trim(), }, }) diff --git a/apps/notifier/.env.example b/apps/notifier/.env.example new file mode 100644 index 0000000..e2fb453 --- /dev/null +++ b/apps/notifier/.env.example @@ -0,0 +1,13 @@ +# Contract for the XCS administrator notification worker. +# Names only; values live in the deployment's secret store. +XCS_NOTIFIER_DATABASE_URL= # held (xcs_notifier role) +XCS_AUTH_ORIGIN= # config; exact HTTPS portal origin +XCS_SMTP_HOST= # config +XCS_SMTP_PORT= # config +# External servers must use `starttls` or `tls`; only local Mailpit may use `disabled`. +XCS_SMTP_TLS_MODE= # config (disabled, starttls, or tls) +XCS_SMTP_USERNAME= # optional held (pair with XCS_SMTP_PASSWORD) +XCS_SMTP_PASSWORD= # optional held (pair with XCS_SMTP_USERNAME) +XCS_SMTP_FROM_NAME= # optional config +XCS_SMTP_FROM_ADDRESS= # config (required for external SMTP) +XCS_SMTP_ENVELOPE_FROM= # optional config (defaults to FROM_ADDRESS) diff --git a/apps/notifier/Dockerfile b/apps/notifier/Dockerfile new file mode 100644 index 0000000..a607b9b --- /dev/null +++ b/apps/notifier/Dockerfile @@ -0,0 +1,39 @@ +# Administrator notification worker. Build context: the repository root. +# It compiles the notifier from apps/web so the worker and web service use the +# exact same mail delivery and database code. +ARG XCS_NODE_IMAGE=node:24-alpine +FROM ${XCS_NODE_IMAGE} AS build + +RUN npm install -g pnpm@10.34.4 + +WORKDIR /workspace +COPY apps/web ./apps/web +COPY db ./db +COPY config ./config + +WORKDIR /workspace/apps/web +RUN pnpm install --frozen-lockfile +RUN NODE_ENV=production pnpm build +RUN pnpm install --frozen-lockfile --prod + +FROM ${XCS_NODE_IMAGE} AS runtime + +ARG XCS_REVISION=unknown +ARG XCS_VERSION=0.0.0 + +LABEL org.opencontainers.image.title="XCS administrator notifier" \ + org.opencontainers.image.description="XCS administrator decision notification worker" \ + org.opencontainers.image.source="https://github.com/XRPL-Commons/XCS" \ + org.opencontainers.image.revision="${XCS_REVISION}" \ + org.opencontainers.image.version="${XCS_VERSION}" \ + org.opencontainers.image.licenses="MIT" + +ENV NODE_ENV=production + +WORKDIR /workspace +COPY --from=build --chown=node:node /workspace/apps/web/dist ./dist +COPY --from=build --chown=node:node /workspace/apps/web/node_modules ./node_modules +COPY --from=build --chown=node:node /workspace/apps/web/package.json ./package.json + +USER node +CMD ["node", "dist/admin/admin-notifier.js"] diff --git a/apps/web/.env.example b/apps/web/.env.example index 4944fe1..c7f68ec 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -1,35 +1,61 @@ -# Contract for the XCS web app (Nuxt server + the /v1 read API). -# Names only; values live in the deployment's secret store. -# -# This file holds secrets and values that genuinely differ per deployment. -# Behavioural settings and feature flags are checked-in code, in -# server/xcs/settings.ts; they are not deployment inputs and a variable of the -# same name here has no effect. +# XCS web deployment contract. Behavioural settings live in +# server/xcs/settings.ts; this file contains only secrets and values that differ +# between deployments. -# Read-only PostgreSQL connection string for the API role. -XCS_DATABASE_URL= # config -# Comma-separated browser origins allowed on /v1/**; `*` is rejected. -XCS_ALLOWED_ORIGINS= # config -# Narrow reverse-proxy CIDRs trusted for forwarded client-IP headers; catch-all /0 is rejected. +# Public API and network. +XCS_DATABASE_URL= # held; xcs_api PostgreSQL URL +XCS_ALLOWED_ORIGINS= # config; explicit comma-separated origins XCS_TRUSTED_PROXY_CIDRS= # optional config -# Bearer token guarding /internal/metrics*; 32 to 256 URL-safe characters. Setting -# it is what turns the two routes on; left unset the routes are not registered at -# all. A value that is not 32 to 256 URL-safe characters fails at start-up rather -# than leaving the deployment quietly unmonitored. -XCS_METRICS_TOKEN= # optional generate -# Salt for hashing pinning client addresses; at least 32 random bytes. Read only -# once XCS_DEMO_PINNING_ENABLED is turned on in server/xcs/settings.ts. -XCS_PINNING_IP_HASH_SECRET= # optional generate -# Path to the network profile JSON this deployment serves; the browser-visible -# profile identifier is that file's `profileId`. Must be the same published file -# the indexer indexes. Left unset, the site does not filter the network list, -# which is what an unset NUXT_PUBLIC_PROFILE_ID used to mean. -XCS_NETWORK_PROFILE= # optional config -# Public wss:// XRPL submission endpoint; no credentials, no private indexer URL. -NUXT_PUBLIC_RPC_URL= # config -# Public Xaman application identifier; enables the Xaman adapter. -NUXT_PUBLIC_XAMAN_API_KEY= # optional -# Deployment origin with a trailing slash, registered in the Xaman Developer Console. -NUXT_PUBLIC_XAMAN_REDIRECT_URL= # optional config -# Public Reown project identifier; enables the WalletConnect adapter. -NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID= # optional +XCS_METRICS_TOKEN= # optional generate; presence enables metrics +XCS_PINNING_IP_HASH_SECRET= # optional generate; demo pinning only +XCS_NETWORK_PROFILE= # optional config; same JSON as the indexer +NUXT_PUBLIC_RPC_URL= # config; public wss:// submission endpoint +NUXT_PUBLIC_XAMAN_API_KEY= # optional public identifier +NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID= # optional public identifier + +# Account workspace. Supplying any field requires the complete group; a complete +# group enables authentication without a separate feature flag. +XCS_AUTH_ORIGIN= # config; exact HTTPS origin +# Local qualification only: 1 requires loopback HTTPS auth and Identity origins. +XCS_LOCAL_IDENTITY_QUALIFICATION= +XCS_IDENTITY_CLIENT_ID= # held +XCS_IDENTITY_CLIENT_SECRET= # held +NUXT_APP_DATABASE_URL= # held; xcs_app PostgreSQL URL + +# Commons review workspace. Supplying its database/key enables it after auth. +NUXT_ADMIN_DATABASE_URL= # held; xcs_admin_app PostgreSQL URL +XCS_ADMIN_DOCUMENT_KEY= # generate + +# Issuer workspace. Supplying its database enables it after auth. The browser +# navigation is derived from this same validated server contract. +NUXT_ISSUER_DATABASE_URL= # held; xcs_issuer PostgreSQL URL + +# Shared private evidence storage. Filesystem is local-only; production uses a +# private S3-compatible bucket such as DigitalOcean Spaces. +XCS_DOCUMENT_STORAGE_DRIVER= # config; filesystem or s3 +XCS_DOCUMENT_FILESYSTEM_DIRECTORY= # local-only absolute private directory +XCS_DOCUMENT_FILESYSTEM_LOCAL= # local-only acknowledgement; must be 1 +XCS_DOCUMENT_S3_ENDPOINT= # config; exact HTTPS origin +XCS_DOCUMENT_S3_REGION= # config +XCS_DOCUMENT_S3_BUCKET= # config; private bucket +XCS_DOCUMENT_S3_PRIVATE_BUCKET= # production acknowledgement; must be 1 +XCS_DOCUMENT_S3_PREFIX= # optional config +XCS_DOCUMENT_S3_ACCESS_KEY_ID= # held +XCS_DOCUMENT_S3_SECRET_ACCESS_KEY= # held + +# Invitation and lifecycle email. External SMTP must use starttls or tls. +XCS_SMTP_HOST= # optional config +XCS_SMTP_PORT= # optional config +XCS_SMTP_TLS_MODE= # optional config; disabled/starttls/tls +XCS_SMTP_USERNAME= # optional held; pair with password +XCS_SMTP_PASSWORD= # optional held; pair with username +XCS_SMTP_FROM_NAME= # optional config +XCS_SMTP_FROM_ADDRESS= # required for external SMTP +XCS_SMTP_ENVELOPE_FROM= # optional config + +# Public hosted payloads. Supplying any hosted value requires the complete +# group and enables publication; the browser origin is derived from it. +XCS_PAYLOAD_DATABASE_URL= # optional held; xcs_payload_writer URL +XCS_PUBLIC_PAYLOAD_BASE_URL= # optional config; exact HTTPS origin +XCS_PAYLOAD_STORAGE_IP_HASH_SECRET= # optional generate +XCS_HOSTED_PAYLOAD_NETWORKS= # optional config diff --git a/apps/web/.prettierignore b/apps/web/.prettierignore index 11a4452..04929d0 100644 --- a/apps/web/.prettierignore +++ b/apps/web/.prettierignore @@ -5,3 +5,4 @@ coverage playwright-report test-results pnpm-lock.yaml +dist diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 7b20825..0a646f5 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -16,6 +16,8 @@ COPY config ./config WORKDIR /workspace/apps/web RUN pnpm install --frozen-lockfile RUN NODE_ENV=production pnpm build +# The optional notification worker uses the same web image and production dependencies. +RUN pnpm install --frozen-lockfile --prod FROM ${XCS_NODE_IMAGE} AS runtime @@ -34,7 +36,11 @@ ENV NITRO_PORT=3000 WORKDIR /workspace COPY --from=build --chown=node:node /workspace/apps/web/.output ./.output +COPY --from=build --chown=node:node /workspace/apps/web/dist ./dist +COPY --from=build --chown=node:node /workspace/apps/web/node_modules ./node_modules +COPY --from=build --chown=node:node /workspace/apps/web/package.json ./package.json COPY --chown=node:node config ./config +RUN mkdir -p /var/lib/xcs-review && chown node:node /var/lib/xcs-review USER node EXPOSE 3000 diff --git a/apps/web/README.md b/apps/web/README.md index 1b8a46a..0a7c4b8 100644 --- a/apps/web/README.md +++ b/apps/web/README.md @@ -1,9 +1,10 @@ # XCS Testnet web app -The Nuxt application is the accountless, non-custodial Testnet site for XCS. Its public navigation -keeps four entries—Explorer, Create, Verify and Docs—while preserving the schema, issuance, -lifecycle and developer workflows in one deployment. EAS and EASScan are interaction-design -references only; the site constructs native XRPL transactions under the frozen XCS v0.1 protocol. +The Nuxt application is the non-custodial Testnet site for XCS. Public read-only discovery remains +accountless; authenticated business mutations use the guided `/issuer` and `/recipient` portals, +and shared attestations open through `/presentations`. The site constructs native XRPL transactions +under the frozen XCS v0.1 protocol while keeping protocol identifiers in integration surfaces, not +business forms. The same Nitro server also **is** the XCS read and verification API: `/v1/**`, `/health/*`, `/internal/metrics*` and `/documentation` are served from this application, on the same origin as the @@ -36,10 +37,10 @@ The site is built on Nuxt UI 4 and Tailwind CSS 4. The XCS identity lives in `EmptyState`); pages compose Nuxt UI primitives with it and keep their workflow logic in `composables/` and `utils/`. -For every write, the application constructs and autofills an XRPL transaction, shows those exact -final fields to the user, then asks an external wallet to sign without submitting. The private key -or seed never enters the application. There is no XCS user or organization account, server session, -team administration or batch issuer in the Testnet beta; one wallet action is prepared at a time. +For every write, the application constructs and autofills an XRPL transaction, presents a readable +business summary, then asks an external wallet to sign without submitting. The private key or seed +never enters the application. Application accounts and organization approval do not replace wallet +authorization. One wallet action is prepared at a time; batch issuance is not implemented. Immediately before opening the wallet, the site requires a fresh, profile-bound readiness proof from the authoritative indexer. It repeats that proof after the wallet returns and before retaining @@ -65,14 +66,15 @@ The current application organizes the existing workflows as follows: statistics and exact search. `/schemas` and `/schemas/:uid` provide paginated schema discovery, `/activity` shows schema registrations only, `/credentials/:generationId` and `/transactions/:hash` expose exact evidence, and `/status` shows the indexer/network view; -- **Create:** `/studio` is the single creation hub. It emphasizes schema registration and - single-credential issuance, then links acceptance, revocation and local operation recovery. - Schema registration has a guided scalar-field editor, course-completion and diploma templates, - plus the advanced JSON editor. Issuance can derive a guided claims form from a compatible - resolved schema and retains the advanced JSON path; -- **Verify:** `/verify` accepts a shared generation ID and opens its exact permalink, or exposes the - complete issuer/subject/schema tuple as an advanced lookup. The permalink reports ledger state, - schema validity, payload integrity and issuer trust separately; +- **Issuer:** `/issuer` derives one next action from application approval, linked issuer wallet, + templates, invitations and issuance state. The guided screens use business names and readable + fields; wallet signing remains explicit; +- **Recipient:** `/recipient` joins claimed invitations, external wallet preparation, attestation + review, acceptance and sharing. Private content is fetched only after the explicit “show and + verify” action; +- **Verify:** `/presentations` removes a bearer fragment before resolving it, opens public shares + without an account, resumes private shares through the one-time login handoff, and leads with + issuer admission, XRPL lifecycle and recipient-wallet proof; - **Docs:** `/developers` loads the active runtime profile, executes the privacy-explicit exact-generation verification flow, derives REST/cURL, TypeScript and monorepo CLI examples from that evidence, documents the wallet `Signer` boundary and catalogs aggregate versus exact API @@ -113,12 +115,16 @@ third-party integrator call. It reads the indexer's PostgreSQL projection as the | `POST` | `/v1/pinning/challenges` | | `POST` | `/v1/pinning/pins` | -The two pinning routes answer `404` unless demo pinning is turned on for the requested profile in -`server/xcs/settings.ts`. It is off in the checked-in settings. +The two pinning routes answer `404` unless `XCS_DEMO_PINNING_ENABLED=true` for the requested profile. +Optional hosted publication adds signed `POST /v1/payloads/:locator` and immutable public reads at +`/p/:locator`. Managed issuer payloads use `/api/issuer/payloads` and `/q/:locator` with separate +application authorization; their bytes are not exposed through projection discovery routes. +SSR calls use Nuxt's in-process request fetch on `/v1`; browsers call the same origin without an +external API base URL. Authoritative ledger-derived routes fail closed: they return `503` when the indexer's writer lease, source agreement, checkpoint, transaction-root evidence or freshness -(`readinessMaxLedgerAgeSeconds` in `server/xcs/settings.ts`, 120 seconds) do not hold. Discovery is deliberately +(`XCS_READINESS_MAX_LEDGER_AGE_SECONDS`, default 120 seconds) do not hold. Discovery is deliberately exact — there is no subject feed, account-wide enumeration or claims search. ### OpenAPI @@ -141,8 +147,9 @@ process — and restrict all three at the ingress to the load balancer and monit ### Metrics -Setting `XCS_METRICS_TOKEN` to 32 to 256 URL-safe characters exposes two routes, both requiring -`Authorization: Bearer $XCS_METRICS_TOKEN`, both `Cache-Control: no-store`, neither consuming the public rate-limit budget: +Supplying a valid `XCS_METRICS_TOKEN` exposes two routes, both requiring `Authorization: Bearer +$XCS_METRICS_TOKEN`, both `Cache-Control: no-store`, neither consuming the public rate-limit budget. +With no token, neither route is registered: - `GET /internal/metrics` — a bounded JSON operator snapshot (schema version 1); - `GET /internal/metrics/prometheus` — the Prometheus exposition Prometheus scrapes. @@ -164,50 +171,81 @@ need a shared edge or store-backed limiter. ## Wallet support -The web app pins `xrpl-connect@1.0.0-rc.0` and registers its eight official adapters: Xaman, -Crossmark, GemWallet, WalletConnect, Ledger, Xyra, Otsu and MetaMask Snap. Crossmark, GemWallet, -Ledger, Xyra, Otsu and MetaMask Snap are registered without deployment-specific configuration. -Xaman is added only when `NUXT_PUBLIC_XAMAN_API_KEY` is set, and WalletConnect only when -`NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID` is set. Both values identify a public application to its -wallet provider; they are browser-visible identifiers, not secrets. One Xaman application key is -used by every visitor of that XCS deployment. Register the deployment origin with a trailing slash -as an exact redirect URI in the Xaman Developer Console. XCS uses that stable root callback from -every page; it never uses the current route as the OAuth callback. - -Two third-party wallet licences are not plain permissive SPDX identifiers, and -`docs/third-party-licenses.md` records why XCS may ship them. The GemWallet permission, granted by -Florent Bouron in January 2026, covers public and beta use of the GemWallet API, which its dual -`LICENSE` otherwise reserves (its MIT-like arm allows only personal, educational and non-public -research use). The WalletConnect Community License Agreement is covered by its attribution and -usage-threshold conditions: attribution is carried by `packages/xrpl-connect/THIRD_PARTY_NOTICES.md` -upstream in `XRPL-Commons/xrpl-connect`, and the XCS maintainer attested on 2026-09-28 that XRPL -Commons is below the agreement's usage thresholds. No CI job verifies any of this, so re-read that -note before a release that changes wallet dependencies. Reclassifying either licence as MIT, -returning to `0.8.2` or deleting the declared dependencies would only hide the same bundled code; it -is not a compliant workaround, and the upstream adapter packages are not yet published separately. - -`vaul-vue@0.4.1` arrives through Nuxt UI's drawer component, reports no licence and ships no licence -file, so `patches/vaul-vue@0.4.1.patch` adds the verbatim upstream MIT text (Copyright (c) 2025 -unovue). Keep that patch: distributing the component without its licence text would breach the MIT -notice condition. - -The workspace applies `patches/xrpl-connect@1.0.0-rc.0.patch` while that release candidate is -pinned. Xaman's OAuth provider serializes `network_id` as a decimal string and may omit optional -network metadata from a later `ping()`, whereas the published adapter requires a complete -string/number pair. The patch accepts only canonical safe-integer IDs, preserves missing optional -metadata and rejects a network that contradicts the explicit transaction target. Because Xaman's -public browser SDK does not expose its supported OAuth `force_network` parameter, the XCS adapter -adds `force_network=TESTNET` only to the official Xaman authorization URL and evicts an older cached -XCS OAuth session when it cannot prove Testnet. The patch also accepts an absent or empty -`multisign_account` in a valid single-signature response: Xaman uses both forms for that -inapplicable field, while the release candidate incorrectly requires an explicit `null`. Every -signing payload separately carries `force_network: TESTNET`, and the adapter validates the resolved -signing network. The patch also retries the authoritative resolved-payload read for sign-only -requests: Xaman can report the signature before that payload is immediately readable, and the RC -otherwise abandons the operation before XCS can validate and submit the blob. Remove these -compatibility measures only after equivalent behavior ships in the pinned dependency. -After a sign request resolves, the XCS adapter closes the script-opened Xaman window when browser -policy allows it and restores focus to the application so submission progress remains visible. +The chooser shows the pending wallet name while approval and network confirmation are outstanding. +Connection errors stay at the top of the scrollable chooser. If the wallet reports another +network, select XRP Ledger Testnet and retry; an existing app permission in the extension does not +by itself establish the site's current session. Reloading the page requires connecting again +because automatic reconnection is disabled. +A connection must finish within 90 seconds; **Cancel connection** invalidates the attempt immediately, +so a late extension response cannot reconnect it. Once approved, the header shows the wallet name, +address, network and an explicit **Disconnect** action. Rejection and timeout errors remain visible. +The open chooser updates when Crossmark emits its real provider-detection event; it does not require +closing and reopening the menu. Account refresh before preparation/signing has a 10-second deadline. +A timeout invalidates the session immediately, even if extension teardown hangs; late replies cannot +restore the old account. + +The rc.2 Crossmark adapter does not forward session-change events. XCS listens to the official +Crossmark SDK's `signout`, `user-change` and `network-change` events. Signout clears the connection; +user/network notifications trigger an authoritative account refresh. Duplicate initialization +notifications retain the approved session when its account and network are unchanged. Changed or +unverifiable sessions are disconnected with an explanation. Pending sign-in events and other wallets +are left alone. This does not replace the account/network and signature checks at submission. + +`pnpm --dir apps/web exec playwright test --config playwright.crossmark.config.ts` checks the full production +connection path against `https://localhost:3443` (override with `XCS_WALLET_TEST_URL`). The test uses +an isolated browser with a fixture of Crossmark's extension messages, including response envelopes +and duplicate initialization notifications; it does not replace SDK methods or modify the deployed +site. Set `XCS_BROWSER_EXECUTABLE` to Brave's executable when using the macOS-trusted local CA. +Actual wallet approval still requires a manual extension test. + +The web app pins `xrpl-connect@1.0.0-rc.2` and +`@xrpl-commons/xrpl-connect-vue@1.0.0-rc.2`. The official Vue plugin owns the application-scoped +`WalletManager` and reactive account state. XCS uses its documented headless `connect()` API from a +small CSP-compatible chooser: the packaged `` loads Google Fonts and inline styles, +which do not satisfy XCS's current browser policy. XCS does not subclass wallet adapters. +One versioned pnpm patch fixes the rc.2 ESM Crossmark adapter used by this application: +it validates the network returned with the fresh SignIn approval instead of discarding it +and reading Crossmark's separately hydrated background replica. Older responses that omit +the network still require the live query. A present but incomplete network is rejected; +Mainnet never becomes Testnet. Account refresh and pre-signature checks validate the documented +SDK session network, updated by approval and network-change events. The raw background getter +is stale in Crossmark 0.2.19; it is not used as a second network gate. Missing/malformed session +data still fails closed, without falling back to the adapter's previous network. See +[`patches/README.md`](./patches/README.md) for scope and removal criteria. +It registers Xaman, Crossmark, GemWallet, WalletConnect, Ledger, Xyra, Otsu and MetaMask Snap. Xaman is added only when `NUXT_PUBLIC_XAMAN_API_KEY` is set, and +WalletConnect only when `NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID` is set. Both values identify a +public application to its wallet provider; they are browser-visible identifiers, not secrets. + +For local testing, keep Otsu and Crossmark in separate browser profiles. The inspected Otsu +unpacked extension locks `window.xrpl` and `window.crossmark`; Crossmark 0.2.19 also writes +those globals. Co-installation can prevent Crossmark's provider from initializing. This is +an extension collision, not something the application should hide by faking availability. + +The pinned XRPL Connect packages declare MIT, but that does not clear every transitive dependency. +The standalone license-policy run reports unresolved `Unknown` entries for +`@gemwallet/api@3.8.0`, `@walletconnect/types@2.25.0` and `vaul-vue@0.4.1`, plus +`nodemailer@10.0.10` (`MIT-0`, outside the current allowlist). Review the installed dependency +artifacts and required notices; no license exception or policy relaxation was added by this port. +Run dependency reports against the standalone lockfile with `--ignore-workspace`. + +`autoConnect` is disabled: an explicit selection in the XCS chooser is the only connection trigger, +so a cached Xaman session is never presented as a fresh user action. Xaman may replace +`LastLedgerSequence` during signing; XCS omits that volatile field from the Xaman request, then +cryptographically checks the complete returned blob and permits only the SDK's explicit +`LastLedgerSequence` refresh rule. The initial wallet network, active XCS profile and configured +XRPL RPC are all required to resolve to Testnet before submission. XCS does not use Xaman's later +optional `ping()` metadata as a second network gate. + +Otsu stores dApp permissions by origin and otherwise resolves a later `connect()` from the stored +address without presenting the wallet approval again. Before an explicit Otsu selection, XCS calls +the adapter's public `disconnect()` method to revoke that stale origin permission, then delegates the +new connection to XRPL Connect. This keeps the selected account bound to a fresh user-approved Otsu +session instead of silently reusing an older address. Otsu's extension service worker can retain the +public account while its signing key is locked. XRPL Connect rc.2 currently mislabels the resulting +Otsu `Account not found` failure as an uninstalled wallet. XCS rechecks the adapter's public +availability, reports the signing-account failure accurately and closes that unusable session. The +user must unlock Otsu and explicitly reconnect before retrying; XCS never accesses wallet keys or +repeats a rejected signing request automatically. XCS filters this surface to adapters that expose `sign()` and never calls `signAndSubmit`. A wallet may return a `tx_blob` or signed `tx_json`; the application normalizes the artifact, derives and @@ -218,9 +256,19 @@ before persistence or relay. Wallet capability is transaction-specific. GemWallet 3.8.x embeds an XRPL validator from before native XLS-70 Credentials: it can sign the `Payment` used for schema registration but rejects -`CredentialCreate`, `CredentialAccept` and `CredentialDelete` before signing. XCS therefore keeps -GemWallet available for schemas while blocking those three transactions early with -`WALLET_CREDENTIAL_TRANSACTION_UNSUPPORTED`. Xaman has native Credential handling in its current +`CredentialCreate`, `CredentialAccept` and `CredentialDelete` before signing. For these three types, +XCS offers an explicit raw-signing path. The full reviewed transaction remains visible, and a new +acknowledgement is required for every attempt: GemWallet displays a hex message, not decoded fields. +This is a transaction authorization, not a login signature. XCS uses `GemWalletAPI.signMessage(hex, +true)`, exported by XRPL Connect, because rc.2's generic message adapter discards the hex flag. +`xrpl.js` validates and serializes the exact prepared transaction, binds its public key to the +connected Testnet account, assembles the returned signature and verifies it through the same SDK +boundary before journaling or submission. No keys enter XCS, no automatic fallback follows a +rejection, and schema Payments still use the normal adapter. Raw signing currently supports master +keys only; delegated regular keys and multisigning are rejected rather than guessed. +See [the real GemWallet Testnet qualification](../../docs/runbooks/gemwallet-raw-signing.md) for +validated create/accept/delete transactions, popup limitations and local rollback. +Xaman has native Credential handling in its current source and is the preferred candidate when its public application identifier is configured, but it still requires the real-wallet Testnet gate below. Unknown adapters remain visible as unvalidated; if one emits the exact legacy `Invalid field TransactionType` error, XCS converts it to the same @@ -236,7 +284,7 @@ signature, persistence and submission invariants, not a released wallet UI. The required Playwright gate exercises schema registration and credential issuance in Chromium, including the two distinct finality stages. Issuance first receives deliberately mismatched indexed -evidence, withholds XCS success, then proves that `/operations` can re-confirm the exact event +evidence, withholds XCS success, then proves that role recovery can re-confirm the exact event without signing or submitting again. The same deterministic journey accepts the pending Credential, removes the resulting active generation through the subject wallet, opens its exact deleted permalink and exports the sanitized `subject_removed` receipt. A separate case proves that rejecting @@ -257,7 +305,7 @@ proves that an external IPFS CID absent from local storage is never described as ```bash pnpm --dir apps/web exec playwright install chromium -pnpm test:e2e +pnpm --dir apps/web test:e2e ``` Playwright starts its own Nuxt development server with `XCS_BROWSER_E2E=1`. In that process only, @@ -301,6 +349,52 @@ Private and public runtime switches must match exactly, and both the Nitro start plugin reject the mode outside a development bundle. `nuxt.config.ts` also rejects `XCS_BROWSER_E2E=1` when `NODE_ENV=production`. +## Schema text fields + +Schema names (1–64 UTF-8 bytes) and descriptions (1–256 UTF-8 bytes) are required +single-line protocol fields. The guided editor uses single-line inputs with byte +counters and field-specific validation messages. Accents and emoji can use several +bytes; the UI does not truncate or replace the submitted text. JSON mode uses the +same protocol checks and identifies the failing field instead of discarding its +error path. + +## Hosted HTTPS payloads + +Supply the complete hosted-payload contract, including `XCS_PUBLIC_PAYLOAD_BASE_URL`; the server +derives the browser-visible origin from that validated value. Hosted issuance requires explicit +public-storage consent and `XCS_LOCAL_PAYLOAD_STORE=0`. +Canonical payloads larger than **64 KiB of UTF-8 bytes** are rejected before preparation and again +before signing, matching the hosted API limit. External issuer-managed storage keeps its own limit. +The page builds the immutable URI, signs and submits `CredentialCreate`, waits for +ledger/indexer confirmation, then publishes the canonical bytes with the validated +signed blob. It checks the publication receipt and reads the HTTPS bytes back. +Before signing, a credential-specific HEAD request checks that the public host is +reachable. A 404 is expected before first publication; network/CORS failures or +server errors block signing. This does not prove future availability or successful +publication. Sharing/acceptance links appear only after exact-byte publication +verification succeeds, never just because the ledger transaction was validated. +Issuer-managed external HTTPS hosting still requires publication before signing. + +The consented public payload is saved in a per-job browser recovery queue before opening the wallet. +The validated signature is persisted before submission and linked to the operation journal. This +link also recovers an interrupted second storage write. Storage/quota failures block submission; +pending copies are never silently evicted. Up to 20 pending copies are retained until successful +publication or explicit removal. Do not clear browser data while recovery is needed. + +After a reload, **Issue** and **Operations** show pending publications. **Retry payload publication** +only uploads the same payload and signed transaction; it never signs or submits another credential. +The API still requires validated, indexed evidence. If submission timed out, reconcile the original +operation in Operations, then retry publication. Recovery is removed only after the receipt and +HTTPS bytes verify; terminal journal signatures are then purged. Damaged records are isolated and +can be explicitly removed without blocking valid jobs. An interrupted wallet approval with no +recorded signature requires checking the wallet/Operations first, not blindly issuing again. +Download the payload copy before removing an unfinished recovery record. Recovery is local to this +browser profile and origin, not a backup across devices; it uses no test payload-serving fallback. + +Do not use a rotating tunnel domain for durable credentials: the URI on XRPL cannot +follow a domain change. A tunnel can prove a disposable Testnet flow, but production +requires a fixed domain and persistent, backed-up storage. + ## Local browser payload store For manual Testnet flows without an issuer-controlled HTTPS host, start the development site with: @@ -328,9 +422,9 @@ when enabled in a production build. ## Browser response security The Node SSR deployment emits defensive browser headers and a nonce-based strict Content Security -Policy in `Content-Security-Policy-Report-Only`. The observed policy has no `unsafe-inline` or +Policy in `Content-Security-Policy-Report-Only`. Scripts do not allow `unsafe-inline` or `unsafe-eval`: Nuxt's server-rendered scripts and styles receive a fresh random nonce for every HTML -response. Production browser connections are limited to the same origin, HTTPS and WSS. Local +response. Inline style attributes remain allowed for Nuxt UI positioning. Production browser connections are limited to the same origin, HTTPS and WSS. Local development additionally permits HTTP and WS for Vite HMR. The CSP is intentionally report-only until every enabled XRPL Connect adapter passes the manual @@ -357,12 +451,7 @@ XRPL WSS endpoint. ## Configuration -The complete contract is [`.env.example`](./.env.example): secrets and per-deployment values only. -Behavioural settings and feature flags -- payload fetching, demo pinning, the metrics routes, the -IPFS endpoints, the readiness threshold and the static issuer trust policy -- are checked-in code in -[`server/xcs/settings.ts`](./server/xcs/settings.ts), with the same parsing and the same error -messages as before. Changing one is a reviewed code change and an image rebuild, and a deployment -variable of the same name has no effect. A minimal deployment sets: +The complete contract is [`.env.example`](./.env.example). A minimal deployment sets: ```bash XCS_DATABASE_URL=postgres://xcs_api:...@db.example:5432/xcs @@ -371,12 +460,13 @@ XCS_TRUSTED_PROXY_CIDRS=10.42.0.2/32 XCS_NETWORK_PROFILE=/workspace/config/networks/testnet.json NUXT_PUBLIC_RPC_URL=wss://s.altnet.rippletest.net:51233 NUXT_PUBLIC_XAMAN_API_KEY=optional-public-xaman-application-id -NUXT_PUBLIC_XAMAN_REDIRECT_URL=https://xcs.example/ NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID=optional-public-reown-project-id ``` `XCS_DATABASE_URL` authenticates as the read-only `xcs_api` role that `db:bootstrap` provisions; the -web app has no migration command and must never receive the admin URL. The database itself is +web app has no migration command and must never receive the admin URL. Optional hosted publication, +auth/admin and issuer services use distinct restricted writer pools configured in `.env.example`; +none may reuse the projection reader connection for writes. The database itself is provisioned outside this repository, and the indexer owns every database command. Forwarded client addresses are ignored unless the immediate peer matches `XCS_TRUSTED_PROXY_CIDRS`. @@ -386,27 +476,17 @@ behind an undeclared proxy. The active network profile is read from this application's own `/v1` API, parsed locally and matched against the RPC server's reported `network_id` before autofill and again before signing or recovery. -This alpha rejects profiles other than XRPL Testnet (`networkId: 1`). - -The browser-visible profile identifier is the `profileId` inside the profile file named by -`XCS_NETWORK_PROFILE` -- the same published file the indexer indexes -- rather than a variable of its -own. The retired `NUXT_PUBLIC_PROFILE_ID` duplicated that value with nothing checking the two agreed, -so a mismatch could have labelled one network's data with another network's name. A named profile is -parsed at start-up and fails closed. If `XCS_NETWORK_PROFILE` is omitted the identifier is empty, and -then exactly one Testnet profile must be available. +This alpha rejects profiles other than XRPL Testnet (`networkId: 1`). The browser-visible profile ID +is derived from `XCS_NETWORK_PROFILE`; if the file is omitted, exactly one Testnet profile must be +available. `NUXT_PUBLIC_XAMAN_API_KEY` and `NUXT_PUBLIC_WALLET_CONNECT_PROJECT_ID` are optional public application identifiers. Omitting either variable removes only that adapter; it does not prevent the other six adapters from loading. Do not put a Xaman secret, WalletConnect relay secret, wallet key or -other credential in either value. `NUXT_PUBLIC_XAMAN_REDIRECT_URL` is optional and otherwise defaults -to the current application origin with a trailing slash. When set, it must be that exact same-origin -root URL; non-loopback deployments require HTTPS. Add the same exact URL to the Xaman application's -redirect allowlist. - -Each independently hosted XCS deployment needs its own Xaman public application key and registered -origin. The Commons key is not a universal credential for arbitrary self-hosted domains. Follow -Xaman's [browser SDK setup](https://docs.xaman.dev/environments/browser-web3); API secrets remain -server-side and are not used by this browser integration. +other credential in either value. Xaman redirect selection is delegated to its SDK; the old +`NUXT_PUBLIC_XAMAN_REDIRECT_URL` variable is no longer consumed. Register the deployment's actual +callback URLs and qualify homepage and deep-link sign-in separately. Each independently hosted +deployment needs its own provider configuration. `XCS_LOCAL_PAYLOAD_STORE` is a source-time development gate rather than a deployment setting. Do not pass it through Compose or expose it as a general Commons storage option. @@ -423,9 +503,10 @@ rather than either private indexer source. ## Pilot payload publication -The deployed Commons beta issuance and acceptance flows support issuer-hosted public HTTPS payloads -only; the loopback development aid documented above is intentionally not deployable. Commons does -not store, cache or search credential claims. The issuer payload host must +The public issuance and acceptance flows support public HTTPS payloads, either externally hosted or +published through the optional signed-upload service above. The ledger projection does not contain +claims; enabled payload services store them separately. Managed issuer private payloads follow the +authorization rules below; the recipient workspace uses a separate authenticated private reader. The payload host must allow the web origin through CORS and return `application/json` (or a `+json` media type). Acceptance first displays only indexed metadata, the URI and its host. The subject must explicitly consent before the browser contacts that host. Consent stays in memory and is bound to the displayed @@ -452,13 +533,12 @@ resolve the URI, so its remote resolver may remain disabled. There is intentiona fetch proxy, avoiding an SSRF trust boundary. IPFS remains part of the protocol and CLI, but is outside the Commons-hosted browser beta. -The standalone `/verify` page opens `/credentials/:generationId` directly when the verifier has a -shared generation ID. Its advanced tuple flow follows the same privacy boundary: it first reads and -displays indexed credential metadata without contacting the issuer. Only after the verifier consents -to the exact displayed host does the browser fetch the payload with credentials omitted, CORS -enabled and redirects disabled, then POST the parsed object to the API. Verification never requests -API-side URI resolution. A link generation constraint is checked before the host fetch and again -before the result is accepted. +The read-only exact Credential page preserves the integration-oriented verification boundary: it +first reads indexed metadata without contacting the issuer. Only after explicit host consent does +the browser fetch the payload with credentials omitted, CORS enabled and redirects disabled, then +POST the parsed object to the API. Verification never requests API-side URI resolution. This is an +Explorer/integrator surface; the guided verifier journey uses `/presentations` and never asks for a +generation ID or tuple. The exact `/credentials/:generationId` permalink applies the same boundary while also showing the schema and bounded lifecycle timeline. Its server-rendered response loads only indexed generation, @@ -483,8 +563,10 @@ exactly equal the reviewed transaction. SDK journal stages are retained with the `LastLedgerSequence`, profile, XRPL result and the minimum business context needed to reconcile the operation. Schema registration stores publisher, canonical schema digest and exact memo size. Issuance stores the tuple, public URI, payload digest and optional expiration. Tuple-only actions -store the exact reviewed generation ID. The `/operations` page first checks transaction status by -hash, but only after decoding the blob and matching its derived hash, `LastLedgerSequence`, account, +store the exact reviewed generation ID. Managed issuer records keep only nonsecret references and +visibility selections locally, never private claims, invitation tokens or email addresses. Role +recovery first checks transaction status by hash, but only after decoding the blob and matching its +derived hash, `LastLedgerSequence`, account, transaction type and any explicit `NetworkID` against the journal. Only a still-unvalidated transaction may be resubmitted, after checking that its generation is still current. The blob is removed from the local record as soon as the operation becomes `validated`, `expired`, or `failed`. @@ -504,7 +586,12 @@ publisher and schema digest for registration; profile and credential tuple for i tuple and generation for every lifecycle action, independently of the selected action. A second recoverable operation with the same key is rejected. Ownership is checked atomically again after the wallet returns the signed blob and before submission. Only a `prepared` draft that has neither hash -nor signed blob may be abandoned to release its key. +nor signed blob may be abandoned to release its key. Managed issuance additionally reserves the +invitation ID across recipient and issuer wallet choices. This survives reloads and excludes concurrent +tabs in one browser profile, including a ledger-successful operation awaiting metadata recovery. +It is not a distributed lock across browsers or devices: simultaneous devices can still sign an +extra ledger object that the application refuses to attach. Backend metadata uniqueness does not +reserve ledger execution. The profile-bound readiness endpoint is checked just before the wallet call and again after the wallet response. Once the SDK has validated the wallet artifact, the application immediately @@ -531,8 +618,8 @@ treated as XCS-confirmed without new exact evidence. Receipts are sanitized local records, not signatures from Commons or independent trust anchors; a verifier must still resolve the referenced transaction against an authoritative XCS indexer. -For any complete operation already validated with `tesSUCCESS`, `/operations` can re-check a -`pending`, `timeout` or `mismatch` business confirmation. Schema registration uses the exact +For any complete operation already validated with `tesSUCCESS`, the originating role screen can +re-check a `pending`, `timeout` or `mismatch` business confirmation. Schema registration uses the exact `GET /v1/networks/:network/schema-registrations/:transactionHash` proof and requires its publisher and canonical digest before accepting the returned UID. Credential issuance and lifecycle actions use `GET /v1/networks/:network/credentials/:issuer/:subject/:schemaUid/events/:transactionHash` and @@ -546,3 +633,51 @@ removal and issuer revocation return to that same generation page so its updated can be reviewed. A generation-bound subject action resolves that generation before the tuple lookup and rejects a connected wallet that is not its subject, instead of surfacing a misleading tuple 404. Reconfirmation never reads a signed blob, connects to XRPL or rebroadcasts the transaction. + +## Optional accounts and issuer workspace + +Authentication is disabled by default. `/account` supports XRP Identity authorization code/PKCE, +PostgreSQL sessions, CSRF-protected mutations, role guards and verified Testnet wallet linking. +Provision the application migrations and dedicated `xcs_app` pool before enabling it. Supported +wallet message formats and real OIDC registration requirements are in the +[authentication runbook](../../docs/runbooks/authentication.md). Public Explorer reads and public +presentation resolution remain accountless; web mutations are role-guided. + +The admin workspace reviews issuer/verifier applications and documents, records decisions and +manages approval/suspension. Approval grants application access, not issuer trust or protocol validity; +admin status alone does not grant access to private credential claims. See +[ADR 0005](../../docs/adr/0005-role-based-application.md). + +The optional issuer workspace provides onboarding, owned schemas, recipient invitations, issuance +and revocation. It reuses the public transaction engine with server-derived organization, schema and +claimant context, a linked issuer wallet and renewed approval checks before signing. The delivery +email is not identity evidence: review the actual authenticated claimant and their verified wallet. +A bearer link can be claimed by any authenticated holder. The recipient inbox provides consented +private acceptance through the same wallet engine and revocable presentation links/QR codes. +Approved verifiers open designated full presentations and retain metadata-only history/CSV. +See the [issuer runbook](../../docs/runbooks/issuer.md) for provisioning and delivery semantics. + +Managed issuance defaults to private. The issuer reviews the complete authorized view and selected +public claims before signing. Canonical bytes stay in private application storage; the immutable +`/q/:locator` URI and digest, issuer/subject addresses and schema remain public on XRPL. Authorized +issuer/recipient sessions may read the full payload; anonymous private views receive only selected +claims. This is access control, not end-to-end encryption. Public disclosure cannot recall copies. +The public reader omits credentials and treats a filtered `x-xcs-claim-scope: public` response as +`PAYLOAD_SCOPE_RESTRICTED`, with integrity not checked, rather than claiming the full digest failed. + +After a confirmed ledger transaction, issuer recovery records its metadata or reconciles revocation +without signing again. Pending or unknown journal outcomes remain blocked. An explicit reset requires +matching durable proof of expiry or a validated failing transaction; it never treats a timeout as +proof that signing is safe again. Browser-local recovery is lost when site data is cleared. Email +notification status is separate from successful ledger recording and cannot undo it. + +See the [recipient/verifier runbook](../../docs/runbooks/recipient-verifier.md) for routes, grants, +login handoff and upgrade order. These workspaces reuse the restricted issuer pool; they add no new +connection URL. Presentation scope and current approval are checked server-side on each disclosure. + +The guided role journey now returns recipients to their inbox after wallet linking and shows +issuers whether the actual claimant has a verified wallet ready for issuance. Every new presentation +requires a separate wallet message signature bound to that exact credential, disclosure and audience. +The verifier opens only the shared link: current organization admission, recipient address/ledger +state and dated holder proof are displayed without manual URI input. Master-key proof limitations, +legacy unsigned links and additive migration 0008 are described in the recipient/verifier runbook. diff --git a/apps/web/app/app.vue b/apps/web/app/app.vue index a0b9d6b..0e3f897 100644 --- a/apps/web/app/app.vue +++ b/apps/web/app/app.vue @@ -1,9 +1,20 @@