diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index fd8823d8..00000000 --- a/.dockerignore +++ /dev/null @@ -1,27 +0,0 @@ -.git -.github -.agents -.claude -.codex -.opencode -.vscode -.idea -node_modules -**/node_modules -**/dist -**/.wrangler -main.js -qa -qa-runs -rca -notes -tests -docs -src -packages/cli -server/tests -server/scripts -server/.env -*.log -*.map -.DS_Store diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3390b319..0cad3f57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,14 +20,10 @@ jobs: - run: npm ci - run: npm ci --prefix server - run: npm run build - - run: npm run build:server-node - run: npm run typecheck:tests - run: npm run typecheck:qa - - run: npm run typecheck:cli - - run: npm run typecheck:server-node - run: npm run guard:no-any - run: npm run guard:production-bundles - run: npm run guard:no-tracked-generated-artifacts - run: npm run test:ci - - run: npm run test:docker - run: npm --prefix server run typecheck diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 857e20c6..9c1c3a15 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,6 @@ permissions: contents: write id-token: write attestations: write - packages: write jobs: release: @@ -29,7 +28,6 @@ jobs: - run: npm run guard:no-tracked-generated-artifacts - run: npm run build:server-release - run: npm run test:ci - - run: npm run test:docker - run: npm --prefix server run typecheck # Sanity check: the tag must match manifest.json version @@ -42,30 +40,6 @@ jobs: exit 1 fi - - uses: docker/setup-buildx-action@v3 - - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and publish server image - id: push-image - uses: docker/build-push-action@v6 - with: - context: . - push: true - build-args: VERSION=${{ github.ref_name }} - tags: ghcr.io/${{ github.repository_owner }}/yaos-server:${{ github.ref_name }} - - - name: Attest server image - uses: actions/attest@v4 - with: - subject-name: ghcr.io/${{ github.repository_owner }}/yaos-server - subject-digest: ${{ steps.push-image.outputs.digest }} - push-to-registry: true - - name: Attest plugin release assets uses: actions/attest@v4 with: @@ -83,7 +57,6 @@ jobs: gh release upload "$GITHUB_REF_NAME" \ yaos.zip \ dist/release-assets/yaos-server.zip \ - dist/release-assets/update-manifest.json \ main.js \ manifest.json \ styles.css \ @@ -94,7 +67,6 @@ jobs: --generate-notes \ yaos.zip \ dist/release-assets/yaos-server.zip \ - dist/release-assets/update-manifest.json \ main.js \ manifest.json \ styles.css diff --git a/.gitignore b/.gitignore index 3e6a592f..b6ea1733 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,6 @@ server/node_modules server/dist server/.env .wrangler -server/.partykit server/.wrangler # Agent/session artifacts diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index cbb3f6cb..00000000 --- a/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# syntax=docker/dockerfile:1.7 - -ARG VERSION=dev -FROM node:24-bookworm-slim AS build -WORKDIR /src - -COPY package.json package-lock.json .npmrc ./ -RUN npm ci --ignore-scripts -COPY server/package.json server/package-lock.json ./server/ -RUN npm ci --prefix server --ignore-scripts - -COPY packages/server-node/package.json packages/server-node/esbuild.config.mjs ./packages/server-node/ -COPY packages/server-node/src ./packages/server-node/src -COPY packages/server-node/vendor ./packages/server-node/vendor -COPY server/src ./server/src -COPY server/vendor ./server/vendor -RUN npm run build:server-node - -FROM node:24-bookworm-slim AS runtime -ARG VERSION -LABEL org.opencontainers.image.title="YAOS Server" \ - org.opencontainers.image.description="Self-hosted schema-6 YAOS synchronization server" \ - org.opencontainers.image.source="https://github.com/kavinsood/yaos" \ - org.opencontainers.image.version="${VERSION}" -ENV NODE_ENV=production \ - YAOS_NODE_HOST=0.0.0.0 \ - YAOS_NODE_PORT=8787 \ - YAOS_NODE_DATA_DIR=/data -WORKDIR /app - -COPY --from=build --chown=node:node /src/packages/server-node/dist ./dist -COPY --from=build --chown=node:node /src/node_modules/ws ./node_modules/ws -RUN mkdir -p /data && chown node:node /data - -USER node -EXPOSE 8787 -VOLUME ["/data"] -HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \ - CMD ["node", "-e", "fetch('http://127.0.0.1:' + process.env.YAOS_NODE_PORT + '/health/ready').then(r => { if (!r.ok) process.exit(1); }).catch(() => process.exit(1));"] -ENTRYPOINT ["node", "dist/server.mjs"] diff --git a/README.md b/README.md index d02a3818..9a797cf0 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,20 @@ **A zero-terminal, real-time sync engine for Obsidian, powered by your own Cloudflare Worker.** -YAOS synchronizes Markdown live across devices and invited people with CRDT merge -semantics. One server can host multiple vaults; each person has a vault-scoped -identity with separately revocable devices. +YAOS syncs your Obsidian vaults live across your devices, with CRDT merging. It is built for one person. Each +server has one operator, and holds any number of vaults. Each device is paired with a one-time code and can be +revoked on its own. YAOS syncing a note across desktop and mobile in real time -No terminal, `.env` file, database setup, or R2 bucket is required for Markdown sync. +No terminal, `.env` file, database setup, or R2 bucket is required for note sync. [![License: 0-BSD](https://img.shields.io/badge/license-0--BSD-green)](LICENSE) ## How it compares -YAOS runs on infrastructure in your Cloudflare account. Markdown content is split into independently loaded CRDT bodies rather than one vault-wide content document. +YAOS runs on infrastructure in your Cloudflare account. The server only relays changes; merging happens on your +devices. | | Conflicts | Real-time | Deployment | No terminal | Free | |---|:---:|:---:|:---:|:---:|:---:| @@ -35,86 +36,109 @@ If you want the official, fully managed experience, use Obsidian Sync. If you wa 1. **Deploy the server.** Click **Deploy to Cloudflare** above. -2. **Claim it.** Open the Worker URL, click **Claim**, and save the operator recovery key. Claiming provisions a Personal vault and returns a one-use owner-bootstrap code. -3. **Install YAOS.** Install the plugin from the Obsidian Marketplace. -4. **Enroll this folder.** Open the setup link or scan the QR code. Use **Add my device** for another installation you own and **Invite person** for a collaborator. - -The operator key opens the server console. It is not a device credential and must not be pasted into plugin settings. - -## Vault collaboration - -Every vault has one owner and any number of full members. Everyone can read and -change the complete vault, including creating, moving, and deleting notes and -attachments. Only the owner manages people, recovery, security audit, vault -policy, ownership transfer, and destruction. YAOS intentionally does not offer -viewer roles, folder permissions, or per-person permission toggles. - -Invitations grant complete plaintext read/write access. Removing a member stops -future admission and closes their sessions, but cannot erase files already -downloaded to their devices. See [vault collaboration](./docs/collaboration.md). - -## Obsidian settings sync - -Settings sync is enabled by default after enrollment and is independent of note sync. Before a pre-existing remote environment can change this folder, open **Settings → YAOS → Obsidian settings sync** and choose **Take the remote seed**, **Seed from this device**, or **Decide initial seed later**. The choice is stored for this exact enrollment, folder, device, and configuration-folder name. Use **Replace remote settings environment** only when this device should replace an existing seed. - -YAOS synchronizes a closed allowlist of Obsidian JSON files, CSS snippets, community-plugin intents, theme intents, and version-matched plugin `data.json`. It does not store plugin or theme binaries; automatic package installation/removal requires separate explicit consent and is off by default. Each person's settings environment is private to their principal and synchronizes only across their own devices. Settings state is stored in the vault Durable Object's SQLite sidecar, not Yjs or R2. Unsupported servers, an off switch, or a detected settings-sync clash pause settings only; notes continue syncing. See [operations](./docs/operations.md#settings-sync-setup-and-operation) for setup, limits, and recovery behavior. - -## Attachments and recovery - -Markdown and explicitly promoted semantic Canvas files use Durable Object SQLite and work without R2. Add a `YAOS_BUCKET` R2 binding to synchronize images, PDFs, opaque Canvas fallback, and other non-Markdown content; enable Canvas promotion/demotion rollback; and enable asynchronous recovery points. +2. **Claim it.** Open the Worker URL and click **Generate a recovery key**. + - Save the operator recovery key in your password manager. It is shown only once, and it is the only way to + sign in to the console. It cannot be reset. + - Tick **I have saved the recovery key**, then click **Claim server**. + - Claiming creates your first vault and shows a one-time pairing code with a QR code. +3. **Install YAOS.** Install the plugin from Obsidian's Community plugins on each device. +4. **Pair a device.** + - On a phone, scan the QR code and tap **Connect Obsidian**. + - On a computer, click **Open in Obsidian**, or open YAOS settings, choose **Pair this device**, and enter the + server URL and the code. + - A code works once and expires after 15 minutes. +5. **Pair more devices.** Click **Pair a device** on the vault in the console. Or, on a device that is already paired, + open YAOS settings and choose **Pair another device**. + +The operator recovery key opens the server console. It is not a device credential; never paste it into plugin +settings. + +## One operator, vaults and devices + +- **One operator.** The person who claims the server is its only operator, and the console at the Worker URL is + theirs. YAOS has no members, invitations or roles. Every paired device can read and change its whole vault. +- **Vaults.** One server holds any number of vaults. Create them in the console with **Create vault**. Each vault + syncs on its own. +- **Devices.** A device stays paired until you revoke it. **Devices** on a vault lists them. + - **Revoke** cuts a device off at once: its connections close, and changes it had not yet synced are dropped. To + use it again, pair it again. + - Revoking cannot erase files already on that device. +- **Reset streams** deletes the vault's synced content on the server and starts it over. You must type the vault + ID first. Devices stay paired and upload again from their files, and attachments are kept. +- **Delete vault** removes the vault, its devices and its attachments for good. You must type the vault ID first. + +## Attachments (optional R2) + +Notes sync through Durable Object storage and need no R2. + +Images, PDFs and other non-note files sync through an R2 bucket bound as `YAOS_BUCKET`. The shipped +`server/wrangler.toml` binds a bucket named `yaos`. Without the binding: +- the server reports attachments as off; +- the console's header line says so; +- the server refuses attachment uploads; +- notes keep syncing. + +Uploads are limited to 10 MiB per file. Reset and restore keep attachments; deleting a vault removes them. Watch the R2 setup video -Recovery points can be captured in the background, browsed by path, and selectively restored. A deployment also needs the included `RecoveryJob` Durable Object binding and migration for this capability. See [operations](./docs/operations.md). - -## Works with local tools +## Restore to a point in time -Obsidian vaults remain ordinary local files. Changes made by editors, scripts, Git tools, or agents enter the same reconciliation path and can synchronize across enrolled devices. +**Restore** in the console rewinds a vault's synced content to any moment in the last 30 days. It uses Cloudflare's +Durable Object point-in-time recovery. +- Device pairings stay as they are now: a restore rewinds content, never access. +- Unused pairing codes are cancelled. +- The vault gets a new epoch, which tells devices that the server content was replaced. +- Restore needs a deployed Worker. Local development (`wrangler dev`) has no point-in-time recovery, and the console + says so. -## Headless Linux client +Separately, each device can keep daily zip snapshots of its notes (YAOS settings, **Daily recovery snapshots**). -The Node 24 CLI synchronizes Markdown in a local directory without Obsidian. It enrolls as its own vault-scoped device; credentials are generated and stored outside the vault rather than copied from another installation. +## Works with local tools -```sh -npm run build:cli +Obsidian vaults remain ordinary local files. Changes made by editors, scripts, Git tools, or agents enter the same reconciliation path and can synchronize across paired devices. -YAOS_HOST=https://sync.example.workers.dev \ -YAOS_PAIRING_CODE=... \ -node packages/cli/dist/yaos.mjs enroll /srv/vault +## Hosting: Cloudflare only -node packages/cli/dist/yaos.mjs daemon /srv/vault -``` +YAOS runs only on Cloudflare: one Worker with two Durable Object classes (a `VaultDO` per vault and one `ConfigDO` per server), plus an optional R2 bucket for attachments. There is no self-hosted Node or Docker server and no headless CLI client. The server core sits behind small storage, socket and clock ports so its tests run on Node; that is a test seam, not a hosting option. -The daemon is Linux/local-filesystem only, Markdown only, and single-process per vault. `.obsidian`, attachments, network filesystems, and rename-identity inference are intentionally outside its contract. See [operations](./docs/operations.md#headless-linux-client). +The server is an opaque relay. It never interprets payloads, checkpoints, stream names or blob contents. -## Self-hosted Docker server +Moving from a pre-rewrite server is a fresh deployment, not an upgrade. There is no data migration: the deploy's Durable Object migration deletes the old classes (`VaultSyncServer`, `ServerConfig`, `RecoveryJob`) together with their stored data, the new server starts unclaimed, and each device re-seeds it from its local files. -The production image packages `packages/server-node`, which runs the same -schema-8 control-plane, vault, settings, attachment, recovery, and deletion -owners as the Cloudflare Worker over Node 24, SQLite, WebSockets, and filesystem -object storage. +## Troubleshooting -```sh -YAOS_PUBLIC_ORIGIN=https://sync.example.com docker compose up --build -d -``` +**YAOS: re-pair device.** The device was revoked in the console, or its vault was deleted. Pair it again with a new +code (**Pair this device**). -The `yaos-data` volume is the complete durable server state. Put TLS in front of the container, preserve that volume, and pin released deployments to an exact `ghcr.io/kavinsood/yaos-server:` image. See [operations](./docs/operations.md#docker-deployment). +**Lost the operator recovery key.** The server stores only a hash of the key, so nobody can recover or reset it. +Paired devices keep syncing, but no one can sign in to the console. -## Troubleshooting +**Restore says the server cannot restore.** Point-in-time recovery exists only on a deployed Worker, and only for the +last 30 days. A time outside that window is refused before anything changes. -**Unauthorized or auth rejected:** The principal membership or this device credential is missing, revoked, or belongs to another vault. Re-enroll with a fresh purpose-correct code. +**Restore incomplete.** The console shows this banner when a restore was interrupted. Pairing, revoking and +resetting that vault wait until it finishes. The server finishes it on its own within about a minute, or you can +press **Restore** to resume it now. -**Recovery unavailable:** Recovery needs both `YAOS_BUCKET` and the `YAOS_RECOVERY_JOBS` binding. Markdown sync remains available without R2. +**YAOS: daily limit.** On the Workers Free plan, Durable Objects allow 100,000 rows written per day. Past that the +server refuses writes until the limit resets; the console shows the reset time. -**Cloudflare deployment issues:** See [operations](./docs/operations.md#troubleshooting), including the required Durable Object migration and R2 binding. +**Attachments do not sync.** If the console's header line says attachments are off, add the `YAOS_BUCKET` R2 binding +and redeploy. Also check **Sync attachments** and **Maximum attachment size (MB)** in YAOS settings. The server +accepts at most 10 MiB per file. -**Files not syncing:** Check exclusions and file-size limits, then use **Show sync debug info**. Safe exports redact the server URL, vault and device identity, credentials, and vault paths. +**Files not syncing.** Check **Excluded paths** in YAOS settings. Then use **Export diagnostics**, which saves a +file without note contents or credentials. If sync seems stuck, use **Rebuild local cache**. ## Engineering documentation -The compact current source set is indexed in [docs/README.md](./docs/README.md). +- [docs/server-rewrite/DECISIONS.md](./docs/server-rewrite/DECISIONS.md): the server design, including its route + table. +- [docs/client-remake/relay-wire.md](./docs/client-remake/relay-wire.md): the wire contract that design amends (see + its section 5). ## License diff --git a/build-server-release.mjs b/build-server-release.mjs index d4158ce9..aee2f777 100644 --- a/build-server-release.mjs +++ b/build-server-release.mjs @@ -1,6 +1,5 @@ import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -9,36 +8,8 @@ const outputDir = resolve(rootDir, "dist/release-assets"); const tempDir = mkdtempSync(join(tmpdir(), "yaos-server-release-")); const serverTempDir = join(tempDir, "server"); -const rootPackage = JSON.parse(readFileSync(resolve(rootDir, "package.json"), "utf8")); -const pluginManifest = JSON.parse(readFileSync(resolve(rootDir, "manifest.json"), "utf8")); const serverPackage = JSON.parse(readFileSync(resolve(rootDir, "server/package.json"), "utf8")); const serverVersionSource = readFileSync(resolve(rootDir, "server/src/version.ts"), "utf8"); -const productVersionsSource = readFileSync( - resolve(rootDir, "server/src/shared/productVersions.ts"), - "utf8", -); -const wranglerSource = readFileSync(resolve(rootDir, "server/wrangler.toml"), "utf8"); -const ywasmSource = JSON.parse(readFileSync(resolve(rootDir, "server/vendor/ywasm/SOURCE.json"), "utf8")); - -function sha256(path) { - return createHash("sha256").update(readFileSync(path)).digest("hex"); -} - -function requirePinnedArtifact(path, expectedHash, expectedBytes, label) { - if (sha256(path) !== expectedHash || statSync(path).size !== expectedBytes) { - throw new Error(`${label} does not match server/vendor/ywasm/SOURCE.json`); - } -} - -function requireRecoveryDeploymentContract(source) { - const binding = /\[\[durable_objects\.bindings\]\][\s\S]*?name\s*=\s*"YAOS_RECOVERY_JOBS"[\s\S]*?class_name\s*=\s*"RecoveryJob"/.test(source); - const sqliteClass = /\[\[migrations\]\][\s\S]*?new_sqlite_classes\s*=\s*\[[^\]]*"RecoveryJob"[^\]]*\]/.test(source); - if (!binding || !sqliteClass) { - throw new Error("server/wrangler.toml must bind YAOS_RECOVERY_JOBS to the SQLite RecoveryJob class"); - } -} - -requireRecoveryDeploymentContract(wranglerSource); function readStringConst(source, name) { const match = source.match(new RegExp(`export const ${name} = "([^"]*)";`)); @@ -47,22 +18,8 @@ function readStringConst(source, name) { } return match[1]; } -function readNumberConst(source, name) { - const match = source.match(new RegExp(`export const ${name} = (\\d+);`)); - if (!match) { - throw new Error(`Unable to read numeric constant ${name} from shared product versions`); - } - return Number(match[1]); -} const serverVersion = readStringConst(serverVersionSource, "SERVER_VERSION"); -const schemaVersion = readNumberConst(productVersionsSource, "SCHEMA_VERSION"); -const storageFormatVersion = readNumberConst(productVersionsSource, "STORAGE_FORMAT_VERSION"); -const protocolVersion = readNumberConst(productVersionsSource, "PROTOCOL_VERSION"); -const snapshotFormatVersion = readNumberConst(productVersionsSource, "SNAPSHOT_FORMAT_VERSION"); -if (schemaVersion !== 8 || storageFormatVersion !== 4 || protocolVersion !== 5 || snapshotFormatVersion !== 4) { - throw new Error("server product versions must remain schema 8 / storage 4 / protocol 5 / snapshot 4"); -} if (serverPackage.version !== serverVersion) { throw new Error( @@ -76,44 +33,18 @@ const serverReleaseOwnedPaths = [ "scripts", "tsconfig.json", "src", - "vendor", ]; const serverReleaseCopyPaths = [...serverReleaseOwnedPaths, "wrangler.toml"]; -const updateManifest = { - latestServerVersion: serverVersion, - latestPluginVersion: pluginManifest.version, - schemaVersion, - storageFormatVersion, - protocolVersion, - snapshotFormatVersion, - deploymentBoundary: "fresh", - releaseNotesUrl: `https://github.com/kavinsood/yaos/releases/tag/${rootPackage.version}`, -}; - +// DECISIONS-GAP: DECISIONS §1 removes update-metadata, so the release no longer emits update-manifest.json (its +// schema/storage/protocol/snapshot pins went with shared/productVersions). The zip keeps its own manifest so the +// in-place updater still refuses a fresh-deployment boundary (server/scripts/update-from-release.mjs). const serverZipManifest = { deploymentBoundary: "fresh", serverVersion, - schemaVersion, - storageFormatVersion, - protocolVersion, - snapshotFormatVersion, updateOwnedPaths: serverReleaseOwnedPaths, - crdtEngine: { - name: "ywasm", - sourceCommit: ywasmSource.commit, - rustToolchain: ywasmSource.rustToolchain, - wasmPack: ywasmSource.wasmPack, - maximumLinearMemoryBytes: ywasmSource.maximumLinearMemoryBytes, - artifactSha256: ywasmSource.artifact.wasmSha256, - }, }; -requirePinnedArtifact(resolve(rootDir, "server/src/crdt/vendor/ywasm/ywasm_bg.wasm"), - ywasmSource.artifact.wasmSha256, ywasmSource.artifact.wasmBytes, "Worker Wasm artifact"); -requirePinnedArtifact(resolve(rootDir, "server/src/crdt/vendor/ywasm/ywasm.mjs"), - ywasmSource.artifact.wrapperSha256, ywasmSource.artifact.wrapperBytes, "Worker ywasm wrapper"); - mkdirSync(outputDir, { recursive: true }); mkdirSync(serverTempDir, { recursive: true }); @@ -127,10 +58,6 @@ writeFileSync( join(serverTempDir, "yaos-server-manifest.json"), `${JSON.stringify(serverZipManifest, null, 2)}\n`, ); -writeFileSync( - resolve(outputDir, "update-manifest.json"), - `${JSON.stringify(updateManifest, null, 2)}\n`, -); const zipPath = resolve(outputDir, "yaos-server.zip"); rmSync(zipPath, { force: true }); diff --git a/compose.yaml b/compose.yaml deleted file mode 100644 index f8a863d5..00000000 --- a/compose.yaml +++ /dev/null @@ -1,28 +0,0 @@ -services: - server: - build: - context: . - dockerfile: Dockerfile - image: ${YAOS_IMAGE:-yaos-server:local} - environment: - YAOS_NODE_HOST: 0.0.0.0 - YAOS_NODE_PORT: 8787 - YAOS_NODE_DATA_DIR: /data - YAOS_NODE_PUBLIC_ORIGIN: ${YAOS_PUBLIC_ORIGIN:-} - ports: - - "${YAOS_BIND:-8787}:8787" - volumes: - - yaos-data:/data - restart: unless-stopped - stop_grace_period: 30s - read_only: true - tmpfs: - - /tmp:size=16m,mode=1777 - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - -volumes: - yaos-data: - name: ${YAOS_DATA_VOLUME:-yaos-data} diff --git a/docs/client-remake/relay-wire.md b/docs/client-remake/relay-wire.md index a809e0db..5deda07b 100644 --- a/docs/client-remake/relay-wire.md +++ b/docs/client-remake/relay-wire.md @@ -53,7 +53,7 @@ fields are legacy (§14). ### 2.2 Claim (fresh server only) -`POST /claim {"operatorRecoveryKey": "<≥32 chars>"}` → `200 {ok, host, vaultId, vaultName, pairingCode, pairingExpiresAt, obsidianUrl, mobileSetupQrDataUrl, capabilities}` +`POST /claim {"operatorRecoveryKey": "<≥32 chars>"}` → `200 {ok, host, vaultId, vaultName, pairingCode, pairingExpiresAt, obsidianUrl, capabilities}` - This creates the server's first vault and returns a one-time owner pairing code (15 min TTL). - If `/api/capabilities` says `claimed: true`, the server already has an operator. Use §2.3 instead. @@ -160,7 +160,7 @@ error followed by close 1006: "maxBinaryMessageBytes": 1049600, "maxTextMessageBytes": 65536, "maxCheckpointBytes": 4194304, "feedDefaultLimit": 1000, "feedMaxLimit": 5000, "readDefaultBytes": 1048576, "readMaxBytes": 4194304, "rateBytesPerSec": 262144, "burstBytes": 2097152, - "groupCommit": { "idleMs": 300, "maxMs": 1500, "maxBytes": 65536, "minIntervalMs": 0 } + "groupCommit": { "idleMs": 300, "maxMs": 1500, "maxBytes": 65536, "minIntervalMs": 1000 } }, "canWrite": true, "principalId": "...", "deviceId": "...", "role": "owner", @@ -595,8 +595,16 @@ change, and some of it is still exercised by `legacy-src/` and the relay2 experi | read page | 4.1 | 238 | | checkpoint put (+GC) | 4.8 | 266 | -Receipt latency is dominated by the 300 ms idle group-commit window, and the rest is the round trip. HTTP routes -pay Worker auth (a device check in the config DO) plus the vault DO hop. +The table measures the server before its rewrite. There, receipt latency was dominated by the 300 ms idle +group-commit window, and the rest was the round trip; HTTP routes paid Worker auth (a device check in the config DO) +plus the vault DO hop. + +The rewritten server (docs/server-rewrite/DECISIONS.md) authenticates devices in the vault DO: no config DO hop +(D2). A commit also waits at least `minIntervalMs` (1000 ms) after the last one (H8). Deployed on the same Worker +(2026-10-06, p50 median of three 31/31 smoke runs), in ms: ticket 91, socket connect 170, peer PROVISIONAL 61, own +receipt 989, peer COMMIT_NOTICE 994, peer COMMITTED 374, ping 64, burst 1150, feed 97, read page 93, checkpoint put +117. Smoke sends each append right after the last receipt, inside `minIntervalMs`. Its first receipt, from idle, takes +372 ms. --- diff --git a/docs/relay2-protocol.md b/docs/relay2-protocol.md deleted file mode 100644 index d1c6182a..00000000 --- a/docs/relay2-protocol.md +++ /dev/null @@ -1,805 +0,0 @@ -# Relay v2 body protocol (spike, `relay-v2-spike` branch) - -Status: experimental. Everything here is gated by the Worker var -`YAOS_RELAY_BODIES === "true"`. With the flag unset or set to any other value, the server is -byte-for-byte the phase0 server: it adds no routes, frames or tables, and nothing on the socket -path changes. - -Owner: server-core agent. The harness agent and reset agent build clients against this document. -Brief: `experiments/PHASE2-RELAY-SPIKE-V2.md`. Binding decisions: `experiments/results/relay2/DECISIONS.md`. - -## 1. Model - -In relay mode the server never holds a CRDT document for a body in the hot path: - -- **Log.** Each accepted update is one row in the existing `vault_journal` (D5 option a). It takes a - `vault_clock` sequence, advances `vault_document_heads`, and writes one catalog event and one - attribution row. Feed, catch-up, bootstrap and pins read that table, so they see relay edits with - no further changes. -- **Merged bytes.** The server keeps the body's merged state as bytes: checkpoint plus journal - tail, merged with ywasm `mergeUpdatesV1`. There is a bounded per-runtime LRU cache of - `{epoch, latestSequence, mergedBytes|null, stateVector}`. - - Bodies whose merged bytes are at most `YAOS_RELAY_EXACT_MERGE_BYTES` (default 256 KiB) are - merged exactly on every append (exact SV, exact no-op check). - - Larger bodies (K3: whole-state merge/SV costs 7-14 ms at 5-10 MB) keep the head SV - **incrementally**: `SV' = pointwise max(SV, SV(update))`. The bytes are marked stale and rebuilt - lazily on the next step1/HTTP read. See §8 for the risk this carries. -- **Byte ops.** State vector and diff are computed from those bytes with the patch-0003 ywasm - byte ops (`server/src/crdt/ywasmByteOps.ts`). No Y.Doc and no `yjs` import on the server. -- **Checkpoint.** An alarm-driven checkpoint merges the tail at the byte level at 50 entries or - 1 MiB. It is written into the existing checkpoint tables in one transaction. One pass merges at - most `YAOS_RELAY_CHECKPOINT_MAX_ROWS` (default 200) tail rows (K3: merge cost is superlinear in - frame count). A longer tail gets a **partial** checkpoint through the 200th row, and the alarm - re-arms immediately for the rest. -- **History GC.** A client performs history GC, holding a lease: `compaction-lease` then - `semantic-reset` (§5). - -Root sockets and root documents are unchanged in relay mode. Only `kind === "body"` sockets and -body documents use the relay path. - -## 2. Socket: accept - -`GET /vault/:id/ws/body/:bodyId?ticket=…&schemaVersion=…&protocolVersion=…`: unchanged URL and -auth. The body epoch comes only from the HMAC socket ticket (`POST …/socket-ticket` with -`bodyEpoch`), as in base. - -Relay-mode differences: - -- **Skipped:** `MAX_BODY_SOCKETS` (32), `cache.admitBody`, `cache.load` and the - semantic-compaction admission pause. -- **Sanity cap:** at most `YAOS_RELAY_MAX_BODY_SOCKETS` (default 5000) body sockets per vault - DO. Over the cap, the upgrade is refused with HTTP 429 `{error:"body_socket_limit"}`. Relay - sockets do not count toward base `MAX_BODY_SOCKETS`. -- **Epoch:** the body epoch is checked against the durable head (`vault_document_heads`). On a - mismatch the server returns the same `SemanticEpochMismatchError` payload as base. -- **Body state:** the body must be active (`activeBodyHead`), as in base. -- **Handshake:** the server sends step1 = the state vector of the stored merged bytes. - `VAULT_READY` is the same frame as base; `durableGeneration` is the head generation. - `capabilities` is the base object plus `relayBodies: 2` - (`{"currentnessQuery":2,"committedHead":2,"relayBodies":2}`). `GET /api/capabilities` also - carries `"relayBodies": 2` when the flag is on, and omits the field when it is off. -- **Runtime identity:** relay body sockets survive a DO wake. The `runtimeEpoch` check in - `message()` is bypassed for them (§4.4). - -## 3. Socket: client → server frames - -### 3.1 Sync frames (binary, y-protocols) - -| Frame | Server action | -|---|---| -| `MESSAGE_SYNC` step1 (client SV) | Reply step2 = `diffUpdate(merged, clientSV)`. Nothing is written. | -| `MESSAGE_SYNC` step2 | Append path (§3.3). | -| `MESSAGE_SYNC` update | Append path (§3.3). | -| `MESSAGE_AWARENESS` | Phase0 semantics: own awareness only; the server rewrites `state.user`. | - -Oversized frames close with 1009, as in base: a frame larger than `MAX_CANDIDATE_BYTES + 64`, -or an inner update larger than `MAX_DURABLE_UPDATE_BYTES` (1,750,000 B). - -### 3.2 Envelope (D6), `__YPS:` text frame immediately before a binary update - -```json -{ - "type": "BODY_UPDATE_ENVELOPE", - "bodyId": "…", - "bodyEpoch": 1, - "clientFrameId": "…", - "payloadDigest": "", - "candidateId": "…", - "candidateDigest": "…", - "contentHash": "", - "size": 1234, - "stateVector": "", - "frameKind": "update" -} -``` - -Required fields: `type`, `bodyId`, `bodyEpoch`, `clientFrameId` and `payloadDigest`. -Optional fields: `candidateId`, `candidateDigest`, `contentHash`, `size`, `stateVector` and -`frameKind` (`"update"` or `"step2"`, informational only). - -Pairing rules: - -- The envelope applies only to the **next binary sync frame** (step2 or update) on the same socket. - - If that frame's `sha256(innerUpdate) !== payloadDigest`, the frame is treated as having **no - envelope**. It is still appended, but gets no `clientFrameId` echo and no hash. - - A second envelope before a binary frame replaces the first. -- An envelope for another `bodyId` or `bodyEpoch` is ignored. - - A binary frame on a socket whose epoch is stale is fenced with 4409 anyway (§4.3). -- Pending envelopes live in DO memory only (G13, kept by design). If the DO hibernates between - the envelope and the binary frame (practically never, since the two are adjacent), the frame - counts as envelope-less: it is appended with an unknown hash and gets no `clientFrameId` echo. - The client times out and resends; the resend is a no-op (`noop: true`) or, with a `candidateId`, - a dedupe hit. A lost envelope can never pair with a different frame, because pairing requires - `sha256(innerUpdate) === payloadDigest`. - -Currentness (D6, invariant #7): `contentHash`/`size` are recorded on the new catalog event -**only if** `stateVector` is present **and** it equals (semantic map equality) the state vector -of the body's merged bytes after the append. - -- Otherwise the new catalog event has `content_hash = NULL, size = NULL`, meaning "unknown". -- Envelope-less frames (e.g. y-partyserver's internal step2) always give "unknown". -- Therefore a recorded hash always describes exactly the merged state at that catalog event. - -Reconciling "unknown": - -- A later frame whose envelope matches the merged SV makes the hash known again. -- HTTP body reads with an unknown hash materialise once (lazy, off the hot path, counted in - diagnostics). They backfill the catalog event in place when it is still the head. -- `BODY_CURRENTNESS_QUERY` answers from the catalog: `contentHash: null` means unknown. The - client reconciles by fetching the body. - -### 3.3 Append path (per binary step2/update frame) - -The path is synchronous in the DO; nothing awaits between the merge and the commit. Steps run in -this order (as implemented): size limit (1009), envelope pairing, 1 empty skip, 2 authority, -3 dedupe, 4 budget, then (inside commit) epoch fence, 5 growth cap, D6 check, 6 transaction, -7 fan-out. The growth cap runs after authority and budget because it needs the merged state; a -revoked or rate-limited socket never costs a merge. Authority runs before dedupe (round 3, G7), so -a revoked device cannot use `candidateId` probes to learn receipts. - -`SYNC_STEP_1` from a body socket is authority-checked the same way before it is answered (round 3, -G6): a revoked actor gets close 4403 instead of a step2 carrying the body. - -1. **Empty skip.** An inner update equal to `[0,0]` is not appended (`emptySkips++`). -2. **Authority re-check.** `validateActor` runs against the DO's own authority tables, with a - per-runtime cache keyed by an in-memory authority version. The version is bumped by device - revocation, authority fences, membership and provisioning writes, and there is a 5 s TTL safety - net. A revoked actor gets `{type:"error", code:"authority_superseded"}` and close 4403. -3. **Candidate dedupe.** This step applies only if the envelope carries `candidateId`. The server - looks up `vault_candidate_receipts(bodyId, deviceId, candidateId)`: - - Same digest (`candidateDigest ?? payloadDigest`): the original receipt is re-sent as - `BODY_COMMITTED` with `deduped: true`. Nothing is appended. - - Different digest: the frame is rejected with - `{type:"BODY_UPDATE_REJECTED", clientFrameId, candidateId, reason:"candidate_id_reused"}`. - Nothing is appended. -4. **Budgets.** Each socket has a token bucket: `YAOS_RELAY_RATE_BYTES_PER_SEC` (default - 262144) refilling a burst of `max(YAOS_RELAY_BURST_BYTES, MAX_DURABLE_UPDATE_BYTES)` (default - and floor 1,750,000 bytes). The floor guarantees one maximum-size frame always fits; before round 2, - frames between 1 MiB and 1.75 MB were rejected forever. An empty bucket gets - `{type:"VAULT_BACKPRESSURE", reason:"relay_rate_limit"}` and close 1013. -5. **No-op skip (growth cap).** Small bodies (exact path): if `mergeUpdates([merged, u])` is - byte-identical to `merged`, the update adds nothing (for example, a reconnect step2 re-sending - a covered delete set). Large bodies (incremental path): only an exact resend of the last - appended update is detected. A skipped update is not appended (`noopSkips++`); the origin still - gets `BODY_COMMITTED` with `noop: true` and the current head sequence. -6. **Commit**, in one `transactionSync` (per frame, or per micro-batch): - - epoch fence against the head - - active-body check against the latest catalog event - - clock +1 - - journal insert (`kind='body'`) - - attribution row(s); a candidate frame's row carries `operation_id = candidateId` and - `request_digest = candidateDigest` (round 3, G14), so `GET /operations/:candidateId/outcome` - finds the commit through the base `committedOperationOutcome` lookup. No - `vault_operation_outcomes` row is written. - - head update (`generation+1`, `latest_sequence`) - - catalog event (copies file/path; new generation; hash or NULL) - - candidate receipt row, only when `candidateId` is present - - There is no head CAS on merged state: appends commute, and only the epoch is compared (D5.1). -7. **After commit:** - - `BODY_COMMITTED` goes to the origin (§4.1). - - The raw update frame is broadcast to the other body sockets of that body and epoch. - - The base `BODY_COMMITTED` goes to root sockets. Peer relay body sockets of the same body and - epoch get it too, with `relay: true, peer: true` and their own admission `runtimeEpoch`, and - are never closed by it. Every origin socket of the commit is excluded (for a micro-batch, - all origins and duplicate senders; round 3, G16), since each already got its own receipt. - - The checkpoint alarm is armed if the tail is over the thresholds. - -**Micro-batching:** `YAOS_RELAY_MICROBATCH_MS` (default `0` = off, clamped to 0..250; the cap -was 50 before round 3). When it is greater than 0, frames arriving within the window share one -transaction and one clock sequence: - -- One journal row holds the merged update. -- There is one attribution row per frame (`mutation_index` = frame index). -- There is one catalog event per batch. -- Every origin gets its own `BODY_COMMITTED`, all with the same `vaultSequence`. -- Batches are keyed by `(bodyId, bodyEpoch)` (round 3, G3). A frame for another epoch never joins - a batch, and the epoch fence judges each batch separately. `semantic-reset` flushes the body's - pending batches before its CAS, so a queued frame either commits first (and the reset sees - `head_advanced`) or is fenced with 4409 after it. -- At flush time every frame's actor is re-validated (round 3, G2). A frame whose actor was revoked - while it waited is dropped, and its socket gets 4403 once (`authorityDrops++`). -- Duplicate `(deviceId, candidateId)` inside one batch (round 3, G11): the first frame is - appended. A later frame with the same digest is not appended and gets the first frame's receipt - with `deduped: true`. A later frame with a different digest gets `BODY_UPDATE_REJECTED` - `candidate_id_reused` (`batchDuplicateCandidates++`). - -## 4. Socket: server → client frames - -### 4.1 `BODY_COMMITTED` (origin, relay mode) - -This is the base `BODY_COMMITTED` frame with extra fields: - -```json -{ - "type": "BODY_COMMITTED", - "bodyId": "…", "bodyEpoch": 1, "vaultGeneration": 1, - "durableGeneration": 17, "vaultSequence": 4242, - "lifecycle": "active", "contentHash": "…|null", "size": 123, - "runtimeEpoch": "", - "relay": true, - "clientFrameId": "…", - "candidateId": "…", "candidateDigest": "…", "payloadDigest": "…", - "commitRuntimeEpoch": "", - "contentHashAccepted": true, - "deduped": false, - "noop": false -} -``` - -- `clientFrameId` is present only when the envelope paired. -- `candidateId` and `candidateDigest` are present only when the envelope carried them. -- `contentHash` and `size` describe the committed head. Clients must compare them with their own - hash before treating the note as current. `contentHashAccepted` says whether this frame's - claimed hash was recorded. -- Durability is guaranteed by ordering: the frame is sent only after `transactionSync` returns. - -### 4.2 Peers - -- Other body sockets of the same body and epoch get the committed update as a normal y-protocols - update frame, after commit. -- Root sockets get the base `BODY_COMMITTED`. Peer relay body sockets get it with - `relay: true, peer: true` and no `clientFrameId` (it is a head notice, not a receipt). -- The origin sockets of a commit never get the peer notice (§3.3 step 7). - -### 4.3 Close codes - -| Code | Meaning | -|---|---| -| 4409 | Semantic epoch fenced. Sent on a frame for a stale epoch and after a `semantic-reset`. Client rebases. | -| 4403 | Authority superseded (revoked device or membership, authority fence). | -| 1013 | Backpressure: token bucket empty, reason `relay rate limit`, preceded by `VAULT_BACKPRESSURE` `relay_rate_limit` (`rateLimitCloses++`). (The socket cap refuses the upgrade with 429.) See the 1013 note below. | -| 1009 | Frame too large. | -| 1008 | Body deleted or not active (`closeBody`), or socket authority mismatch (root sockets only). | -| 1011 | Durable commit failed (`commitFailures`), or `relay frame error`: any throw before the append (`frameErrors`, preceded by `VAULT_ERROR` `relay_frame_error`). The client reconnects and resends through step2. | - -**1013: ours vs the platform (round 4).** The server sends 1013 only with one of these reasons: -`relay rate limit` (relay token bucket), `semantic compaction pressure`, `body cache budget exceeded` -and `pending durability budget exceeded` (base socket service). None of them is `Service overloaded`. -A 1013 with reason `Service overloaded` (or an empty reason) comes from the Cloudflare runtime: -the Durable Object was overloaded (too many queued requests or events, or CPU saturation), and -the platform shed the connection. The server never sees these closes, so no counter moves. To tell -them apart, check the reason string and whether `rateLimitCloses` grew (and whether a -`VAULT_BACKPRESSURE` frame arrived first). The K1 1013 was the platform: the sender was flooding -one DO at mb=0 with one transaction and one output-gate flush per frame. - -**No silent drops (round 4).** Every non-empty `SYNC_UPDATE` frame (`updateFrames`) ends in exactly -one outcome. Each outcome has a counter, and the counters satisfy - -`updateFrames = appendFrames + noopSkips + dedupeHits + dedupeConflicts + batchDuplicateCandidates -+ authorityCloses + authorityDrops + rateLimitCloses + epochFences + bodyInactiveCloses -+ tooLargeCloses + commitFailures + frameErrors` - -Each outcome is one of: appended, acked or skipped as a justified no-op, or the socket is closed -with an error code. A throw anywhere before the append, including the micro-batch timer path and -`VaultSocketService.relayMessage`, closes 1011 (`failFrames`); before round 4 the socket path sent -`VAULT_ERROR` and kept the socket open, so the frame was lost with no close. A throw after the -append (`postCommitErrors`, `lastFrameError`) is counted and the remaining post-commit steps -still run, so one failing origin send can never cost a peer its fan-out frame. - -### 4.4 Runtime identity - -Base clients accept `BODY_COMMITTED` only when `runtimeEpoch` equals the epoch that admitted -their socket. Relay body sockets survive a DO wake, so relay mode does the following: - -- `runtimeEpoch` echoes the socket's **admission** epoch (from its attachment), so existing parsing - keeps working. -- `commitRuntimeEpoch` names the runtime that actually committed. -- The receipt's evidence is commit-before-send plus `vaultSequence`, not runtime identity. -- Relay body sockets of an older runtime receive `BODY_COMMITTED` normally; they are not closed. -- Root sockets keep base behaviour (hint or close). - -## 5. HTTP - -All new routes exist only when `YAOS_RELAY_BODIES === "true"` (404 otherwise), and use the -normal vault bearer auth. - -### 5.1 `POST /vault/:id/body/:bodyId/compaction-lease` - -Requires the `vault.content.write` capability. - -Request: `{"expectedEpoch": 1, "ttlMs": 120000}`. `ttlMs` is optional, defaults to 120000 and is -clamped to [1000, 600000]. `{"release": ""}` releases a lease the caller holds -(`200 {"released": true}` or `404 {"released": false}`). - -This is a CAS on `relay_compaction_leases(body_id PK)`. The lease is granted if there is no row, -the row has expired, or the row is held by the same device (re-grant, new `leaseId`), **and** -`expectedEpoch` equals the head epoch. - -- `200 {"granted": true, "leaseId", "expiresAt", "epoch", "headSequence", "generation", "stateVector", "policy"}` - - `headSequence` is the head `latest_sequence`. - - `stateVector` is the base64 SV of the merged bytes at `headSequence`. - - `policy` (round 2) is `{"lastResetAt": ms|null, "cooldownMs", "cooldownRemainingMs", "nextResetAllowedAt": ms|null}`. -- `429 {"granted": false, "reason": "cooldown", "epoch", "headSequence", "policy"}` with a - `retry-after` header in seconds. This is the server reset cooldown (round 2): no lease is granted - until `YAOS_RELAY_RESET_COOLDOWN_MS` (default 24 h, mirroring - `BODY_COMPACTION_THRESHOLDS.softCooldownMs`) has passed since the body's last semantic reset. - `lastResetAt` is `vault_semantic_compaction_state.last_compacted_at`, which - `semanticResetFromEncodedState` already writes, so the cooldown adds no table. `semantic-reset` - re-checks the cooldown, so a lease granted just before a concurrent reset cannot slip through. - Test workers set the var small in `[vars]`. -- `409 {"granted": false, "reason": "held"|"epoch_mismatch", "epoch", "headSequence", "holderDeviceId"?, "expiresAt"?}` -- `403 {"granted": false, "reason": "authority_superseded", "epoch": null, "headSequence": null}` - (round 3, G19): the actor is re-validated right before the lease CAS, not only at request - start. - -Leases are released when authority goes away (round 3, G19). Device revocation deletes every -lease that device holds. An authority fence deletes the leases of every device and principal -subject it names. This is conservative: a fence over a principal drops the leases of all that -principal's devices. -- `404 {"granted": false, "reason": "not_found", …}`: the body is unknown or not active. - -**Client shape change (round 2, `scripts/relay2/reset/leaseClient.ts` + `httpTransport.ts`).** A -granted result gains `policy`. A denied result gains `reason: "cooldown"` (HTTP 429) and an -optional `policy`; the client should wait `retry-after` or `policy.cooldownRemainingMs` rather than -retry. The SV coverage check is gone (§5.2), so the builder's `coverStateVector` GC-struct prefix is -no longer needed; it is still accepted. Transports should switch to the octet-stream reset body. - -### 5.2 `POST /vault/:id/body/:bodyId/semantic-reset` - -Requires the `vault.content.write` capability. - -Two request shapes are accepted. - -**Binary (preferred, round 2).** `Content-Type: application/octet-stream`. The body is the raw -Yjs v1 full-state snapshot, up to `MAX_CATCH_UP_BYTES` (8 MiB). The metadata goes in headers: - -| Header | Query fallback | Value | -|---|---|---| -| `x-yaos-lease-id` | `leaseId` | lease id | -| `x-yaos-expected-epoch` | `expectedEpoch` | integer ≥ 1 | -| `x-yaos-covered-sequence` | `coveredSequence` | integer ≥ 0 | -| `x-yaos-content-hash` | `contentHash` | 64 lowercase hex | -| `x-yaos-content-bytes` | `contentBytes` | integer ≥ 0 | - -A header wins over the query parameter of the same field. The binary shape skips base64 (4/3 -inflation) and JSON parsing, which is about 1.8x the snapshot in transient JS heap, on a 128 MB -isolate. - -**JSON (legacy, kept).** The forwarded body limit is 8 MiB, so the snapshot tops out near 6 MB: - -```json -{"leaseId": "…", "expectedEpoch": 1, "coveredSequence": 4242, - "contentHash": "…", "contentBytes": 1234, "snapshot": ""} -``` - -The server checks, and then installs, in order. Before step 1 it flushes the body's pending -micro-batches (§3.3). Authority is re-validated inside the reset transaction (round 3, G19); the -snapshot upload before it can take minutes. A revoked actor gets -`403 {"ok": false, "reason": "authority_superseded"}` and its leases are deleted. - -0. **Structural sanity only:** the snapshot has between 2 bytes and 8 MiB, its state vector - decodes (`stateVectorFromUpdate`), and the SV is non-empty unless `contentBytes == 0`. - Otherwise the server returns `400 invalid_snapshot`. - - **Round 2 removed the SV coverage check** (`SV(snapshot) ⊇ headSV`). A correct client - snapshot is *lineage-fresh*: a new Y.Doc holding the text, with a new client id. Its SV - never covers the old head SV, so the check rejected every correct reset unless the client - prepended GC structs. And it proved nothing about content anyway. -1. The lease is valid (same `leaseId`, same device, not expired). -2. `expectedEpoch` equals the head epoch. -3. `coveredSequence` equals the head `latest_sequence` **exactly**. -4. The reset cooldown has passed (`409 {"reason": "cooldown", "policy"}`; see §5.1). - -**Currency proof.** Steps 1–3 together are the proof that the snapshot is current: epoch CAS, -plus a lease, plus `coveredSequence == head` exactly. Appends take a `vault_clock` sequence inside -their commit transaction, and the reset runs in one synchronous transaction that re-reads the head. -So if any edit landed after the head the client built from, the head moved and the reset fails -with `head_advanced`. The server cannot check that the snapshot *text* equals the head text -without a document. That part stays client trust, like the envelope hashes (D6). - -**Hot-note CAS starvation.** On a note receiving edits continuously (for example, a live -transcript), the head can move during every build → upload window, so every reset fails -`head_advanced` until the lease expires. Mitigations: - -- *Client (recommended):* re-read the head, rebuild from the local - doc (the rebuild is O(text), about 100 ms at 5 MB), re-POST under the same lease, and add - jittered backoff between attempts. Only reset when the note has been idle for a moment, since - the policy thresholds make resets rare anyway. -- *Server (documented, not implemented):* an optional append-pause window. While a lease is held - with `pauseAppends: true`, relay appends for that body are deferred (queued in memory, bounded - by the lease TTL and by `YAOS_RELAY_MICROBATCH_MS`-style buffering) or rejected with - `VAULT_BACKPRESSURE`. That gives the holder a quiet window of at most a few seconds. It trades - edit latency for guaranteed progress, and the lease TTL bounds the damage if the holder dies. -4. `semanticResetFromEncodedState(bodyId, snapshot, {throughSequence: coveredSequence, generation, semanticEpoch: expectedEpoch})` - installs the snapshot. In one transaction it writes a checkpoint at a new sequence, bumps the - epoch, advances the head and copies the catalog event. In relay mode that catalog event - records the client's `contentHash` and `contentBytes`. -5. The lease row is deleted. -6. `fenceSemanticEpoch` closes old-epoch sockets with 4409. -7. The merged-bytes cache is invalidated. - -Responses: - -- `200 {"ok": true, "epoch", "previousEpoch", "sequence", "generation", "fencedSockets", "policy"}` -- `409 {"ok": false, "reason": "lease_invalid"|"lease_expired"|"epoch_mismatch"|"head_advanced"|"cooldown", "epoch", "headSequence", "policy"?}` -- `403 {"ok": false, "reason": "authority_superseded"}` (round 3) - - `head_advanced`: rebuild from the current head and retry while the lease is valid. -- `400 {"ok": false, "reason": "invalid_request"|"invalid_snapshot"}` -- `413 {"ok": false, "reason": "too_large"|…}`: the body is over the 8 MiB bound. - -A successful reset also drops any resident base-path document (`discardResident`) and prunes the -old epoch's journal rows, checkpoints and manifests (`pruneUnpinnedDocumentHistory`; bootstrap and -restore pins are kept). - -The server does **not** verify that `contentHash` matches the snapshot text, because it would need -a doc. Clients are trusted, as with envelope hashes. - -### 5.3 Existing reads, relay mode - -- **`GET /vault/:id/body/:bodyId`** returns the merged stored bytes (checkpoint + tail, byte - merge). Headers: - - `x-yaos-body-epoch` - - `x-yaos-generation` - - `x-yaos-head-sequence` (the head `latest_sequence`) - - `x-yaos-content-hash`, empty when unknown - - `x-yaos-size`, empty when unknown - - `x-yaos-content-hash-state`: `known`, `materialised` (computed just now by the lazy hash) or - `unknown` - - It returns `413 {"error": "relay_merge_budget_exceeded"}` for an over-budget body (§6.2). -- **`HEAD /vault/:id/body/:bodyId`** (relay mode only; base has no HEAD route) returns the same - headers with no body. It reads only the durable head and the catalog: no byte merge and no - document. `x-yaos-content-hash-state` is `known` or `unknown`. A client polls it cheaply for - sequence, generation and epoch. -- **`/catch-up`** returns merged stored bytes. `contentHash`/`size` can be `null` (unknown). An - over-budget body gets a per-item `{"status": 413, "error": "relay_merge_budget_exceeded"}`. -- **Bootstrap body reads** (`/bootstrap/:id/body/:bodyId` and the batch route) use a byte merge in - relay mode (round 2). They merge the pinned checkpoint at or before the boundary with the - journal tail through the boundary (`durableMergedBytes(bodyId, boundarySequence)`), with no - document. A reset after the boundary still serves the old lineage, because the pin keeps the - old epoch's rows (this is what the WIP and committed code do; an older revision of this doc - said bootstrap reconstructs a document, G17). Over budget, the batch route returns - `413 {"error": "relay_merge_budget_exceeded", "bodyId", "overBudgetBodyIds"}` before it reserves - any budget (round 3, G20). An unmodified client bisects on 413, so it ends up failing only on - the over-budget body; the extra merges of the other bodies in the rejected batches are bounded - by the bisection depth. -- **Lazy hash.** If the head catalog event has a NULL hash and the merged bytes are at most - `YAOS_RELAY_LAZY_HASH_MAX_BYTES` (default 3 MiB), the server materialises the document once, - computes `canonicalMarkdownHash` and backfills that catalog event in place - (`materialisations++`). Above that size the hash stays unknown (`lazyHashSkips++`), because a - document for a large, struct-dense body can exceed the wasm memory budget (§6.2). - A per-runtime cache (round 3, G4; at most 256 bodies) remembers the computed hash per - `(bodyId, epoch, headSequence)`, so repeated reads of a head whose catalog event is no longer the - newest one do not materialise again (`lazyHashCacheHits++`). A reset clears the body's entry. - -### 5.4 Diagnostics - -- `GET /vault/:id/diagnostics` (existing, `vault.diagnostics.read`) gains a `relay` object: - - ```json - { "enabled": true, "config": {…}, - "counters": { "appends", "appendFrames", "emptySkips", "noopSkips", "dedupeHits", "dedupeConflicts", - "materialisations", "checkpoints", "lastCheckpointMs", "checkpointRowsWritten", - "resets", "leaseGrants", "leaseDenials", "rowsWritten", "rateLimitCloses", - "epochFences", "authorityCloses", "commitFailures", "mergedCacheRebuilds", - "step2Replies", "envelopeMismatches", "hashAccepted", "hashUnknown", - "incrementalAppends", "stateVectorDrift", "partialCheckpoints", "mergeBudgetRejects", - "unmergedStep2Replies", "lazyHashSkips", "lazyHashCacheHits", "authorityDrops", - "batchDuplicateCandidates", "residentStaleSkips", "floorAdvances", "floorRowsPruned", - "updateFrames", "frameErrors", "postCommitErrors", "bodyInactiveCloses", "tooLargeCloses", - "leanCatalogEvents", "leanCoalesceRowsWritten", "checkpointsFromCache", - "appendsPerSecond" }, - "lastFrameError": { "at", "message" } | null, - "documentMaterialisations": { "root", "nonRoot", "recentNonRoot": [ { "documentId", "throughSequence", "at" } ] }, - "byteOps": "", - "bodies": [ { "bodyId", "epoch", "latestSequence", "generation", "logRows", "logBytes", - "checkpointSequence", "stateVectorBytes", "mergedBytes|null", "stateVectorExact" } ], - "mergedCacheBytes": 0, "pendingEnvelopes": 0, "pendingBatches": 0, - "staleResidents": 0, "overBudgetBodies": 0, - "vaultJournalRows": 123, - "ywasmLinearMemoryBytes": 0 } - ``` - - Counters are per runtime; they reset when the DO is evicted. `vaultJournalRows` is a - `COUNT(*)`; it runs only here and in tests (round 3, G15), never on the append or alarm path, - and the feed floor (§6.3) keeps the table bounded. `documentMaterialisations` counts - every `VaultDocumentStore.reconstructDocument` call, relay or base, for any reason. In relay mode - `nonRoot` should move only for the paths listed in §6.1. - -- `GET /vault/:id/debug/relay-table-counts`, gated by `YAOS_TEST_ONLY_DEBUG_ROUTES === "true"` AND - the relay flag, with the operator session: `{tables: {name: rowCount}}` for every table. - -## 6. Environment variables - -| Var | Default | Meaning | -|---|---|---| -| `YAOS_RELAY_BODIES` | unset | `"true"` enables everything in this doc. | -| `YAOS_RELAY_MICROBATCH_MS` | `0` | Micro-batch window (ms); 0 means one tx per frame. Clamped to 0..250 (round 3; was 50). | -| `YAOS_RELAY_MAX_BODY_SOCKETS` | `5000` | Sanity cap on body sockets per vault DO. | -| `YAOS_RELAY_RATE_BYTES_PER_SEC` | `262144` | Token-bucket refill per socket. | -| `YAOS_RELAY_BURST_BYTES` | `1048576` | Token-bucket capacity per socket. | -| `YAOS_RELAY_MERGED_CACHE_BYTES` | `16777216` | Per-runtime LRU budget for merged body bytes. | -| `YAOS_RELAY_CHECKPOINT_ENTRIES` | `50` | Tail-entry threshold that arms the checkpoint alarm. | -| `YAOS_RELAY_CHECKPOINT_BYTES` | `1048576` | Tail-byte threshold that arms the checkpoint alarm. | -| `YAOS_RELAY_EXACT_MERGE_BYTES` | `262144` | Bodies up to this merged size use the exact per-append merge; larger ones use the incremental SV. | -| `YAOS_RELAY_CHECKPOINT_MAX_ROWS` | `200` | Max tail rows merged per checkpoint pass (partial checkpoints beyond). | -| `YAOS_RELAY_RESET_COOLDOWN_MS` | `86400000` | Minimum ms between two semantic resets of one body (lease → 429 `cooldown`). `0` disables. | -| `YAOS_RELAY_MAX_MERGE_INPUT_BYTES` | `9437184` | Max summed input of one server byte merge (§6.2). | -| `YAOS_RELAY_LAZY_HASH_MAX_BYTES` | `3145728` | Bodies above this never get the lazy-hash materialisation. | -| `YAOS_RELAY_LEAN_ROWS` | unset | `"true"` enables lean rows (§6.4). | -| `YAOS_RELAY_LEAN_CATALOG_DELAY_MS` | `2000` | Lean mode: the relay alarm runs this long after the first pending append; it coalesces catalog events and checkpoints. Range 0..600000. | -| `YAOS_BULK_CREATE_MAX_ITEMS` | `500` | Create-bulk item cap (files + attachments) per request; clamped to 1..500 (can only lower). Not relay-gated. | -| `YAOS_BULK_CREATE_MAX_BYTES` | `4194304` | Create-bulk byte cap over all file frames per request; clamped to 65536..4194304. A single-file batch keeps the per-note ceiling. Not relay-gated. The effective caps are advertised as `bulkCreate: {maxItems, maxBytes}` in `/api/capabilities` and in every create-bulk 413 body; the client splits by them (byte budget = 7/8 of `maxBytes`) and still halves on a 413 without caps. | - -Local tests: `YAOS_TEST_FORCE_RELAY_BODIES=true` (or `YAOS_TEST_RELAY_BODIES=true`) makes runtimes -built without an env (unit suites) default to relay mode (`relayBodiesTestDefault()` in -`server/src/relayFlag.ts`). An explicit `YAOS_RELAY_BODIES` in the env always wins. The local -wrangler launchers (`tests/conformance/launch/wrangler.ts`, `tests/headless/wrangler.ts`) forward -`YAOS_RELAY_BODIES` from the process env as `--var`, and the Node host reads it from -`process.env`. - -### 6.1 Remaining document materialisations in relay mode - -These code paths still build a server Y.Doc for a relay body. Each is counted in -`documentMaterialisations`: - -| Path | When | Why it stays | -|---|---|---| -| Lazy hash (`bodyHttpState`) | HTTP GET/catch-up of a body whose head catalog hash is NULL, merged bytes ≤ 3 MiB | This is the only way to get text from bytes without a doc. It is at most once per unknown head, then backfilled. Clients avoid it by sending D6 claims. | -| HTTP candidate path / create (`vaultLifecycleService.createDocument`, candidate commit) | Closed-file edits and file creates go through the base HTTP candidate path, which loads the body into the document cache | The markdown → CRDT diff (and validation) needs a document. Relay covers only open-editor socket edits. Moving closed-file edits to "client builds the update, server appends" is a client change. | -| Dirty-doc `writeLiveCheckpoint` | A body resident in the cache (from the candidate path) that is dirty gets a live checkpoint from the doc | This belongs to the base cache lifecycle. The relay alarm pass skips dirty residents; the base pass handles them. | - -Relay appends never apply an update to a resident document (round 3, G5). `syncDocumentCache` -discards a clean resident. A dirty or validating resident is left alone, counted in -`residentStaleSkips`, remembered, and evicted by the next alarm pass once it is clean. That -resident can be behind the durable head until then; the next reader reloads from durable state. -| Semantic docs / base compaction | Server semantic compaction | Disabled for relay bodies (§7.9). It can still run for bodies created before the flag. | -| Root | All root reads and writes | Out of scope: the relay covers bodies only. | - -Paths that do **not** materialise in relay mode: socket accept (step1 = the SV of merged bytes), -appends, step2 replies, checkpoints, GET with a known hash, HEAD, catch-up with known hashes, -bootstrap body reads (round 2), lease and reset (structural check only). The unit test -`HTTP reads: …` asserts this with the counter. - -### 6.2 Memory (ywasm byte ops) - -Byte ops are stateless: every op copies its inputs into ywasm linear memory, decodes them, and -encodes the result. The linear memory grows up to a 96 MiB cap (`ywasmByteOps`) and **never -shrinks**, inside a 128 MB isolate. - -Measured with `scripts/relay2-core-reset-loop.ts` on Node, the same wasm build: - -| Input | Op | ywasm peak | -|---|---|---| -| 10 MB single-lineage text (checkpoint + 211 tail parts) | merge (15 ms) + SV (6 ms) | 39–48 MiB | -| 8.77 MB bloated 5 MB note (`reset-fixtures/sized-5m.update`, 246k structs) | SV only | 54 MiB | -| same | `merge([state, 50 tail edits])` | 85 MiB | -| same | `diff(state, emptySV)` | 88 MiB | -| same | SV + merge + diffs, sequentially | 95.6 MiB | -| same | `merge([state, state])` | **trap (`unreachable`, OOM)** | - -The rule of thumb is about 4.5x input bytes for sparse text and up to about 10x for struct-dense -updates. - -**Critical: after an OOM trap the ywasm instance is poisoned.** Every later op, even a 100-byte -merge, fails with an allocation error until the isolate restarts. In the DO that would take down -every relay body of the vault until eviction. Re-instantiating the module after a trap is owned by -the ywasm agent. The server side now **refuses before calling into wasm**: - -- `YAOS_RELAY_MAX_MERGE_INPUT_BYTES` (default 9 MiB) bounds the summed input of every server - merge: `durableMergedBytes` throws `RelayMergeBudgetError`. The 9 MiB default sits above the - 8 MiB `MAX_CATCH_UP_BYTES` response cap, so any body a GET could return is still merged. A - dense 9 MiB input (about 90 MiB) is at the edge; lower the budget if the ywasm agent measures - denser fixtures. -- An over-budget body keeps `bytes: null` and an exact SV (the pointwise max of the per-part SVs; - each part is one checkpoint or one ≤1.75 MB frame). Appends keep working. - - **Step1** is answered with the stored parts, one `SYNC_STEP_2` per part, unmerged and undiffed - (`unmergedStep2Replies++`). y-protocols clients apply them idempotently. - - **GET/catch-up/bootstrap** return 413 `relay_merge_budget_exceeded`. - - **The checkpoint** is bounded (round 3, G20). When the tail is over - `YAOS_RELAY_CHECKPOINT_MAX_ROWS` or the checkpoint plus the tail is over the merge budget, it - merges the longest tail prefix that fits (`tailPrefixBounded`) and writes a partial - checkpoint. When not even one row fits, the body is marked over budget in - `relay_body_budget(body_id, body_epoch, input_bytes, marked_at)` (`mergeBudgetRejects++`). The - marker survives DO eviction, so the alarm stops re-arming for that body. It is valid only - while the head epoch matches, and a semantic reset clears it. -- Exact per-append merges only run for bodies ≤ `YAOS_RELAY_EXACT_MERGE_BYTES` (256 KiB), which - is far below the budget. - -### 6.3 Feed floor (round 3, G1) - -Before round 3 relay bodies were never pruned from `vault_journal`: the base maintenance loop -skipped them and nothing advanced the feed floor, so the table grew by one row per append forever. -Now the alarm calls `runCheckpointPass`, which: - -1. evicts stale residents (§6.1); -2. checkpoints relay bodies over the thresholds, skipping dirty residents and over-budget bodies; -3. sets `floor = min(currentSequence - 1000, pinBoundary - 1 for every active pin)` and calls - `advanceFeedFloor(floor)` when it is above the current floor. That prunes journal rows at or - below the floor that a complete checkpoint covers. - -Bootstrap and restore pins are respected (the floor stays below every pin boundary). The base -maintenance loop now runs only for non-relay bodies. Measured in the unit suite: 5,000 relay -appends leave 1,001 journal rows (floor 4002, 41 floor advances, 2.45 s total). - -**5 MB bloated note: no server doc is needed.** Server compaction is disabled for relay bodies. -The reset takes the client's snapshot and runs only the structural SV parse. GET, step2 and the -checkpoint use byte ops. The lazy hash is skipped above 3 MiB. The bloated fixture's SV-only -parse, 54 MiB, is the largest single op on the reset path. - -**Repeated-reset slowdown (live: 5.0 s → 20.9 s → 116 s `lease_expired`).** A local loop with a -5 MB note, 10 iterations, real SQLite and the real route -(`node tests/run-typescript.mjs --test-aliases scripts/relay2-core-reset-loop.ts --mb 5 --n 10`) -showed nothing growing on the server: - -- Each reset takes about 140 ms with the binary body and about 155 ms with JSON, flat across - iterations. -- The post-reset read takes about 132 ms. -- Table counts stay flat (journal 2, checkpoint chunks 3, manifests 1). Old epochs are pruned. -- ywasm stays at 20.2 MiB and the heap at about 45 MiB. - -So server storage and CPU do not grow per reset. The likely live cause is isolate memory -pressure: ywasm memory near the cap from the pre-round-2 `snapshotCoversHead` full-state rebuild -and SV, plus the base64 JSON body (about 3x the snapshot in transient heap), plus the 16 MiB -merged LRU, on a 128 MB isolate. That slows GC until the isolate is reset. Client or network time -could also contribute. Round 2 removes the full-state rebuild on reset and adds the binary body. - -Live re-measurement after round 2 (`scripts/relay2-core-live-reset.ts --mb 5 --n 6` against -`yaos-relay2-coresmoke-1`; the 5 MB note is grown by 1 MB relay appends; binary 5.24 MB snapshot -each time; results in `results/relay2/core2-live-reset-5mb.json`): - -- Six consecutive resets all returned 200: 3.7, 3.3, 3.6, 1.7, 3.1 and 3.1 s. That time is - dominated by the 5 MB upload from the test machine. There is no growth and no `lease_expired`. -- The GET after each reset took 1.2–1.7 s for 5.24 MB, with `hashState: known`. -- HEAD took 180–260 ms. -- With the 1 s test cooldown, an immediate re-lease got `429 cooldown` (`retry-after: 1`) once. In - the other five iterations the reset round-trip had already outlasted the cooldown. - -### 6.4 Lean rows (round 4, `YAOS_RELAY_LEAN_ROWS=true`) - -Rows written per append by the default relay transaction are listed below. "Node" is -`node:sqlite` `rowsWritten`, which counts table rows only. "CF" is Cloudflare `rowsWritten`, -which also counts index entries. - -| Write | Node | CF | Lean | -|---|---|---|---| -| `vault_clock` UPDATE (sequence) | 1 | 1 | removed | -| `vault_journal` INSERT (PK + `(document_id, sequence)` index) | 1 | 2 | kept | -| `vault_mutation_attribution` INSERT | 1 | 1–2 | frame 0 inline on the journal row; extra micro-batch frames keep a row | -| `vault_document_heads` UPDATE | 1 | 1 | kept | -| `vault_catalog_events` INSERT (PK + `(body_id, sequence DESC)` index) | 1 | 2–3 | coalesced by the alarm | -| `vault_candidate_receipts` INSERT (candidate frames only) | +1 | +2 | kept (G13) | - -The deployed numbers come from `scripts/relay2-core-smoke.ts`, which measures 20 acked edits per -frame kind using the diagnostics `rowsWritten` delta: - -- `yaos-relay2-coresmoke-1` (default): 9 CF rows per plain append, 11 per candidate frame. -- `yaos-relay2-corelean-1` (lean): 3 per plain append, 5 per candidate frame. - -Minimum is 2 node rows or 3 CF rows per plain append (journal + index + head), and 3 node or 5 CF -rows for candidate frames. The default is 5 node / 9 CF rows (6 / 11 with a candidate). -Amortised over a pass (`scripts/relay2-core-alarmprobe.ts 750 30`, one pass per 50 appends): -lean 2.13 node rows per append vs 5.07 by default. The coalescing pass writes 3 rows per dirty body -(clock sync, clock advance, event). - -How lean mode works: - -- **Sequence.** An append reads `MAX(vault_clock, MAX(vault_journal.sequence)) + 1` and does not - write the clock. Every other allocator (`clockAdvanceSql`) and `currentSequence()` use the same - MAX, so sequences stay unique and monotonic. Every journal `DELETE` first raises the clock to the - journal max (`syncLeanClock`), so pruning can never free a sequence for reuse. -- **Attribution.** Frame 0's actor, `operationId` and `requestDigest` are stored in nullable - `attr_*` columns on the journal row (added by `ALTER TABLE` in `enableLeanRows`). Frames 1..n of a - micro-batch keep their `vault_mutation_attribution` rows. `committedOperationOutcome` falls back - to the inline columns (G14 still holds). -- **Accepted hash.** The hash goes to `relay_content_hash` / `relay_size` on the journal row. Lazy - backfill updates it in place. -- **Catalog.** No per-append event. `getCatalogHeadAt` / `listCatalogAt` overlay the newest body - journal row after the latest catalog event, at or before the boundary: its sequence, generation - and inline hash. Readers of the current head (HTTP reads, rename, bootstrap manifests, - `currentBodyHead`) are exact. The relay alarm is armed on every lean append, delayed by - `YAOS_RELAY_LEAN_CATALOG_DELAY_MS`. At the start of `runCheckpointPass` it writes one event per - dirty body (`coalesceLeanCatalog`), with the head generation and the head row's inline hash, or - NULL when that row's hash is unknown. The feed floor does not advance if coalescing failed. - `semanticReset` coalesces the body first, because the reset event copies the latest one. - -What each removed write costs: - -1. **Clock write.** Nothing observable. The cost is a MAX subquery on every allocation and on - `currentSequence()` (a PK tail lookup), plus a clock sync before journal deletes. Turning the - flag off again requires one coalescing pass first (not automated in the spike). -2. **Attribution row.** Frame 0's attribution now lives and dies with its journal row. It is - pruned at the feed floor, not kept forever. A candidate resend older than about 1000 sequences - (and older than its receipt) is re-appended. On the exact path that is a CRDT no-op; the - receipt table (kept) covers the normal window. -3. **Catalog event.** The raw catalog log (`catalogDeltaAt` delta feed, recovery-authority raw - queries, historical catalog at boundaries between coalesced events) lags by up to the - coalescing delay. Intermediate generations and hashes never appear as events: one event per - body per pass. Coalesced events carry no attribution (`mutation_index` 0, no operation). Body - hash changes reach peers through the delta feed up to about 2 s later. Socket fan-out is - unaffected. - -Verdict (go/no-go input). A plain append drops from 9 to 3 CF rows. It cannot go lower without -dropping the head update (the head is the fence and the merged-cache key) or the journal index. -Candidate frames cost 5 CF rows. The catalog lag is the only semantic change a client can see. - -### 6.5 mb=0 tail latency (round 4) - -At mb=0 every frame is its own `transactionSync`, and fan-out waits for the output gate (the SQLite -flush). Latency therefore follows commit latency, not CPU. The p90 of 0.4–2 s (vs about 68 ms at -mb10) comes from head-of-line blocking behind the checkpoint alarm. The alarm arms when the tail -reaches `YAOS_RELAY_CHECKPOINT_ENTRIES` (50) rows. mb=0 makes 1 row per frame instead of 1 per -batch, so it arms about 3x more often. Each pass is a durable byte merge plus a checkpoint write -plus a floor advance: about 60 ms CPU, with 0.5–4 s wall time deployed. The alarm runs -single-threaded with the DO, so every frame that arrives meanwhile queues behind it, and output -gates add a flush per frame. - -Round 4 cheap fix: a full checkpoint uses the in-memory merged bytes when they are at the head -(`checkpointsFromCache`). That skips the SQLite read and the wasm merge; local checkpoint time -fell from 3.3 to 1.8 ms per pass for a 30 KiB body. Lean mode also delays and coalesces the alarm. -The remaining lever is micro-batching (mb 5–10). It turns N frames into one transaction and one -flush, and is the recommended default. - -## 7. Design decisions - -1. **Log storage in `vault_journal`** (D5 a): one source of truth, so feed, catch-up, bootstrap and - pins keep working unchanged. -2. **No head CAS; epoch fence only** (D5.1). Concurrent appends never fail with - `document_head_changed`. -3. **Merged bytes, not a Y.Doc.** A per-runtime LRU holds bytes and the SV. It is rebuilt from - SQL (checkpoint + tail, byte merge) when the head sequence moved. A lazy materialisation - happens only for an unknown hash on HTTP reads, and is counted. -4. **Growth cap by byte-equality of merge** (small bodies). `mergeUpdates([M,u]) == M` holds exactly - when `u` adds nothing (verified by probe for covered inserts, full states, delete-only updates - and `[0,0]`). The cost is one merge, which the exact path already needs. Large bodies only - catch exact resends (K3 cost). -5. **Currentness requires an SV match** (D6). A hash is recorded only if the client SV equals the - merged SV after the append (invariant #7). -6. **Synchronous frame handling** (sha256 in JS, no awaits), so envelope pairing and appends never - interleave across frames. -7. **The receipt is commit-before-send**, not runtime identity (§4.4). -8. **Dedupe uses `vault_candidate_receipts`.** `vault_operation_outcomes` is not written per - append (write amplification). The outcome route still works through the attribution row's - `operation_id` (round 3, G14). Receipt pruning runs in the checkpoint alarm, not per append. -9. **Server-side semantic compaction is disabled for relay bodies.** Lease-based client reset - replaces it; `recordCommit` is skipped for relay appends. - -## 8. Deviations (from brief / base behaviour) - -- **Validation.** There is no server-side frontmatter or markdown-size validation on relay - appends, because it would need a doc. The per-frame byte limit still applies. -- **Pending envelopes are memory-only** (§3.2, G13, not fixed on purpose). Hibernation between the - envelope and its frame means an envelope-less commit with an unknown hash and no echo. The - client resends; the resend is a no-op or a dedupe hit. Persisting envelopes would cost one row - write per frame for a case that practically never happens. -- **Checkpoint at `YAOS_RELAY_CHECKPOINT_*`** uses the byte merge. Checkpoints of relay bodies - never touch the document cache. -- **`vault_operation_outcomes` is not written** for relay appends (G14). Since round 3 candidate - frames are still retrievable through `GET /operations/:candidateId/outcome`, via the attribution - row's `operation_id`. Envelope-only frames without `candidateId` have no operation id. -- **The reset lease is at least as strict as needed**: `coveredSequence` must equal the head - exactly, so a busy note can starve resets. See §5.2 for the hot-note mitigations. -- **Incremental SV on large bodies (K3).** For bodies over `YAOS_RELAY_EXACT_MERGE_BYTES` the head - SV is the pointwise max of the old SV and the update's SV. If an update arrives with a causal - gap (its structs cannot integrate yet), that max overstates what the merged state contains, - so a D6 hash claim can be accepted when it should not be. Well-behaved y-protocols clients send - causally complete updates, so this should not happen in practice. Every rebuild or checkpoint - compares the incremental SV with the exact one and counts mismatches in - `counters.stateVectorDrift`; it was 0 in all tests. - - Round 3 (G8): a rebuild that could not merge (over the exact size or the budget) marks the SV - `stateVectorExact: false`, and D6 accepts a hash only while the SV is exact. So for bodies - over 256 KiB **a client hash claim is never accepted**: the hash stays unknown and is filled - by the lazy hash (up to 3 MiB). Above 3 MiB it stays unknown until a semantic reset records the - client's hash. This trades hash availability for never recording an overstated claim. The growth cap on large bodies only catches - exact resends, so a covered-but-different re-send (e.g. a reconnect step2) of a large body is - appended (it is idempotent, and costs one journal row). -- **Bootstrap body reads use a byte merge** (round 2; §5.3). The bootstrap catalog page can carry - `contentHash: null` for relay appends whose hash was not accepted; clients must treat that as - unknown. -- **Over-budget bodies** (§6.2) are not served by GET, catch-up or bootstrap (413). They are - checkpointed only as far as a bounded prefix fits, and then marked (persisted) until a client - reset shrinks them. -- **Queued frames of a closed socket still commit** (G2). Re-validation at flush checks - authority, not socket liveness; the frame was received before the close, and the sender resends - anyway. -- **Reset flushes pending batches first** (G3). A reset that races queued frames can therefore fail - with `head_advanced` where, without batching, the frames would have been fenced instead. -- **Lease release on fence is conservative** (G19): every lease of every listed device and - principal subject is deleted. -- **Dirty residents are left stale** (G5) instead of having relay updates applied in-process. -- **Unused helper:** `RelayBodyStore.tailPrefixSequence` is superseded by `tailPrefixBounded` and - is kept only until the spike is folded. -- **Checkpoints are bounded** to `YAOS_RELAY_CHECKPOINT_MAX_ROWS` rows per pass (partial - checkpoint + immediate alarm re-arm) instead of one merge of the whole tail. - -(Updated as implementation lands; see also `results/relay2/STATUS.md`.) diff --git a/docs/relay3-group-commit.md b/docs/relay3-group-commit.md deleted file mode 100644 index 10a62391..00000000 --- a/docs/relay3-group-commit.md +++ /dev/null @@ -1,254 +0,0 @@ -# Relay v3: group commit, tail row, receipt ring - -Status: experiment, behind `YAOS_RELAY_GROUP_COMMIT` (`1` or `true`). It needs `YAOS_RELAY_BODIES=true` -and `YAOS_RELAY_LEAN_ROWS=true` and is ignored without them. With the flag off, the schema and every -code path are the same as relay v2. - -## Why - -Relay v2 lean costs about 5–6 Cloudflare rows written per keystroke. Each frame pays for: - -- a journal insert plus its index entries -- a head update -- a `vault_candidate_receipts` insert plus its autoindex -- part of a `setAlarm` - -The Free plan allows 100k rows a day. v3 aims for about 3 rows per *commit* and commits about once per -typing burst. - -## Design - -**B1. Share now, save in groups, confirm after saving** (`RelayBodyService.groupEnqueue` / `flushGroup`). - -- Every message of a relay socket is first charged to the socket's rate budget on its raw size (R12, below). - Then a frame is checked as in v2: authority, envelope and receipt dedupe. -- After the check, the frame is broadcast at once and buffered under its (body, epoch). -- The buffer is committed in one `transactionSync` when the first of these happens: - - `YAOS_RELAY_GC_IDLE_MS` passes with no new frame (default 300) **and** at least - `YAOS_RELAY_GC_MIN_INTERVAL_MS` (default 1000; 0 = off) has passed since the body's previous commit - - `YAOS_RELAY_GC_MAX_MS` passes after the first frame (default 1500) - - the buffer reaches `YAOS_RELAY_GC_MAX_BYTES` (default 65536; this flush is synchronous) - - a forced flush: HTTP candidate/currentness read, authority fence, semantic reset (immediate) -- Commit-rate cap (commit 4a34f56): the idle timer is re-armed on every frame at - max(last frame + idle, previous commit + min interval). A deferred idle commit is therefore scheduled - for previous commit + min interval; it never waits for the next frame. Idle commits of a body are at - least 1 s apart whatever the typing rhythm, and every frame still waits at most `GC_MAX_MS`. Without - the cap, typing with gaps over 300 ms (1–3 keys/s, mobile, pauses) committed once per frame. -- Both timers use `setTimeout`. v3 never calls `setAlarm` per commit. -- At flush, these checks run again: - - G2 revocation: the frame is dropped and the socket closed with 4403 - - the epoch, in memory and again inside the transaction: the frame is fenced with 4409 - - B6 receipt dedupe: a resend whose first copy was committed in the meantime gets a dedupe ack - - in-buffer duplicate candidates - - the growth cap: a covered update on a large body gets an exact merge check, so wake-resync step2s - and resends write nothing -- `BODY_COMMITTED` receipts use the v2 wire format. They are sent only after the commit returns. -- After a commit there is no second fan-out. -- A socket that runs step1 while frames are buffered gets the buffered frames right after its step2. - The frames are not durable yet, so that step2 does not contain them. - -**B2. One tail row per body.** - -- `relay_body_tail` is WITHOUT ROWID with a text primary key. It stores the group-commit records - `(sequence, generation, update)` since the body's last checkpoint, plus the last accepted hash and - the last committer. -- Reads merge journal rows and tail records by sequence (`journalUpdatesAfter`): reconstruct, - `durableParts`, bootstrap byte reads and step1. -- When the tail reaches `YAOS_RELAY_GC_TAIL_BYTES` (64 KB) or 512 records, the server writes a - byte-merged checkpoint at the head (`checkpointTail`). In the same transaction: - - the tail row is deleted - - first, any record that an active history pin still needs moves to a journal row -- The catalog event for the body is coalesced first, so the accepted hash survives. -- A tail that would grow past 1 MiB (a body whose checkpoint cannot make progress) falls back to a v2 - lean journal row. - -**B3. One receipt row per device.** - -- `relay_device_receipts` is WITHOUT ROWID with `client_id` as the primary key. -- It holds a newest-first JSON ring of the last 256 receipts, with the same 15-minute TTL as v2. The - row is UPSERTed in place once per commit per device. -- `candidateReceipt` and `committedOperationOutcome` (G14) fall back to the ring. -- A resend whose receipt has aged out still reaches the growth cap. It is a CRDT no-op and gets a - no-op ack. It never writes a row. - -**B4. Schema trims.** - -- Neither v3 table has a secondary index. -- The head is updated once per group commit. -- The sequence is `MAX(clock, journal head, tail head) + 1` (`leanHeadSql`). The clock is raised - before any tail row is deleted, so a sequence is never reused. - -**Feed and catalog.** - -- `listChangesAfter` reports a body whose head has no journal row as one `body` entry at the head - sequence. The feed means "document changed at S", so the intermediate tail sequences collapse into - the head. -- The catalog head is served by the lean overlay, which reads the tail hash. -- The catalog *delta feed* is published by coalescing. That happens in three places: - - at a tail checkpoint - - at a semantic reset - - in the relay alarm pass. A commit arms that alarm at most once per `YAOS_RELAY_GC_CATALOG_DELAY_MS` - window (default 30000; 0 turns it off), and the host dedupes it in memory. - -**Wake re-sync** (`ensureWakeResync`). It is called from fetch, webSocketMessage and alarm. - -- A new runtime sends step1 once, with the durable state vector, to every open relay body socket that - an earlier runtime admitted, if that socket's epoch is still current. -- Each client replies with a step2 of whatever the server lacks. This recovers frames that a crashed - runtime broadcast but never committed. -- A step2 that is already covered writes nothing (growth cap). - -## Invariant changes - -1. **"Durable before broadcast" becomes "durable before receipt".** Peers can apply a frame that is - later lost in a crash. The origin never received a receipt for it, so it resends; wake re-sync also - pulls it back from any peer. HTTP reads (GET/HEAD, bootstrap, feed) only ever see durable state. -2. **Revocation fence (R11).** Peers and durable state must agree across a revocation, and no frame - received after the fence is ever broadcast or appended. Every authority write in the DO - (`/__yaos/authority-fence`, `/__yaos/revoke-device-sockets`) first calls - `relay.flushForAuthorityFence()` in the same turn, with no await in between. Frames relayed before - the fence (already broadcast) are committed and acked under the authority that relayed them. The - write bumps `authorityVersion` synchronously, so the next frame of the revoked device misses the - actor cache (the 5 s TTL never serves a stale "allowed" across a bump). It fails the fresh check at - receipt (4403) and is never broadcast or buffered; later frames of that socket hit - `failedSocketDrops`. There is no cross-isolate window: one DO instance per vault holds both the - relay buffers and the authority tables, and it is single-threaded. Safety net: if a buffered - frame's device has lost authority at flush (an authority writer that skipped the fence flush), - v3 commits it anyway, because it was authorised at receipt and peers already applied it. It is - counted in `revokedBroadcastCommits` and the socket is closed with 4403. v2 micro-batching - broadcasts only after the commit, so it still drops such frames (`authorityDrops`). - (Before d3b4db8, the deployed B4 run broadcast 10 frames of a revoked device and then dropped - them at flush, so peers diverged from durable state.) -3. Tail attribution only covers the last commit: `attr_*` holds the last committer. No per-frame - `vault_mutation_attribution` rows are written. G14 outcomes come from the receipt ring. -4. Receipts are bounded to 256 per device. An older resend is answered as a CRDT no-op. -5. No silent drops: `updateFrames` = the sum of the v2 outcomes + `groupDropped`. `groupFlushDedupes` - is a subset of `dedupeHits`. On v3 no frame is dropped by authority at flush (item 2), so - `authorityDrops` stays 0. `revokedBroadcastCommits` is a subset of the commit outcomes. - `failedSocketDrops` is also an outcome (item 6). -6. **Acks are cumulative-safe per socket** (the client treats an ack for frame N as confirming every - earlier frame of that socket). Frames of one socket are committed in arrival order: one body and one - epoch per socket, one buffer per (body, epoch), synchronous byte-cap flushes, one transaction per - group. A refused frame (rate limit, too large, authority, epoch fence, inactive body, commit failure, - frame error, including one frame of an over-limit split) fences its socket: frames that arrived after - it are dropped unacked (`failedSocketDrops`), and frames buffered before it still commit and ack. - After a wake, an earlier-runtime socket's acks are held until its re-sync step2 is durable - (`wakeHeldAcks`), because a frame it sent before the wake may have been lost with the buffer. -7. An HTTP candidate or a currentness query for a body flushes that body's buffer first - (`groupFlushReads`). Neither ever waits on a timer. An HTTP copy of relayed bytes is therefore a - CRDT no-op: it writes no body rows, only its receipt ring entry (1 row). -8. **HTTP save through the group-commit store.** A non-creation HTTP candidate (`POST - body/:id/candidate`, or one item of `/body/candidates`) is still validated by - `VaultCandidateService`: ywasm apply, canonical markdown and the exact content hash. It is then - committed by `relay.commitHttpCandidate`: tail 1 + head 1 + receipt ring 1 = 3 rows. The base path - would write clock, journal, attribution, head upsert, catalog event, operation outcome and - candidate receipt (14 rows). A candidate that changes nothing writes only its ring entry (1 row). - The receipt JSON is unchanged. The ring answers replay, digest reuse - (`candidate_id_reused_with_different_digest`) and `GET operations/:id/outcome`, matching on - principal, membership revision and credential revision. The ring is bounded (256 per device, TTL - as before). Because CRDT re-application is idempotent, a replay older than the ring re-validates - as a no-op. The exact hash is claimed only if no commit landed between validation and commit; - otherwise the hash is left unknown and materialised lazily. The catalog event is coalesced by the - alarm, as for relay frames. Creation candidates and merged updates over the durable value limit - keep the base path. No client change is needed. - -## Rows per commit (Cloudflare billing, inferred) - -| Statement | Rows | Why | -| --- | --- | --- | -| `relay_body_tail` UPSERT | 1 | WITHOUT ROWID, text primary key, no index | -| `vault_document_heads` UPDATE | 1 | sequence, generation and hash columns; no indexed column changes | -| `relay_device_receipts` UPSERT | 1 per device with candidate frames in the commit | WITHOUT ROWID, no index | -| `setAlarm` | at most 1 per 30 s window per DO | catalog coalescing | - -That is 3 rows per commit for one typing device. At the defaults, a typing burst of N keystrokes -costs about 3 rows instead of about 5N. Each tail checkpoint (one per about 64 KB of updates) also -writes about 12 rows: checkpoint chunk + manifest, catalog event, clock, and the tail delete. -`tests/server/relay3-group-commit.ts` measures all of this as statement changes × (1 + index -entries). - -HTTP candidate POST (closed-note save), measured locally with the same meter. These numbers are -inferred and have not been run on a deployment: - -| Path | Rows per POST | By table | -| --- | --- | --- | -| Base (before) | 14 | clock 1, journal 2, attribution 2, head 2, catalog event 3, operation outcome 2, candidate receipt 2 (deployed gql: 13.82) | -| v3 relay path (after) | 3 | tail 1, head 1, receipt ring 1 | -| v3, identical re-save | 1 | receipt ring 1 | -| Deferred catalog coalesce | 5 per body per 30 s window, plus 1 alarm | clock 2, catalog event 3 | - -For the HTTPSAVE shape (one POST every 5 s to one closed note), that is (6 × 3 + 5 + 1) / 6 ≈ 4.0 rows -per POST. Tail checkpoints add about 12 rows per 512 records or 64 KB. - -Typing rhythm, one body, one device, 30 s per pattern. Local virtual-time accounting -(`tests/server/relay3-group-commit.ts`, "commit-rate cap"), inferred and not deployed. Rows are -3 per commit (tail, head, receipt ring) with candidate ids: - -| Pattern | Rows/keystroke without cap | With cap (1000 ms) | Max commits/s (without / with) | Receipt p50 / max with cap | -| --- | --- | --- | --- | --- | -| 1 key/s | 3.00 | 3.00 | 1 / 1 | 300 / 300 ms | -| 2 keys/s | 3.00 | 1.55 | 2 / 1 | 800 / 800 ms | -| 3 keys/s | 3.00 | 1.03 | 3 / 1 | 633 / 967 ms | -| 5 keys/s | 0.38 | 0.38 | 1 / 1 | 900 / 1,500 ms | -| 8 keys/s | 0.25 | 0.25 | 1 / 1 | 875 / 1,500 ms | -| Bursty, gaps 100–2000 ms | 2.61 | 2.23 | 2 / 1 | 300 / 1,362 ms | - -At or below 1 key/s the cap cannot help: one commit per keystroke is already one per second, so the -cost is 3 rows per keystroke (3 rows per second of typing). The cap bounds rows per second of activity -at 3 per body, not rows per keystroke. - -## Rate limit (R12, commit 4a34f56) - -Why B7-v3 closed the flooder late (code reading; the deployed timings are measured, the cause is -inferred): - -- The bucket was checked at step 4 of `handleSyncFrame`. Before it, every frame paid: attachment - parse; for text, `JSON.parse` of the envelope (envelopes were never charged); outer and sync - decode; SHA-256 of the whole update when an envelope was pending; the authority check; and a - `candidateReceipt` SQL read when the envelope had a candidate id. -- It charged `update.byteLength` only. Envelopes, step1, empty frames and dedupe hits (replays) - were free. -- After the 1013, the close takes time to complete, and frames already queued kept arriving. - Each one still paid the decode and the digest before the failed-socket check. On v2 there was - no failed-socket mark at all. -- Each accepted 64 KiB frame hits the 64 KB bytes cap, so it triggers a synchronous group commit - and often a tail checkpoint (a merge of a growing body). The DO is single-threaded, so it - processed the flooder's queue more slowly than the queue filled. The bucket measures processing - time, so the overdraft frame was reached only after the backlog: 43 s and 63 MB in a3. In a1, - Cloudflare shed the DO (`1013 Service overloaded`) first. -- The bucket is per socket and starts full on every new socket. - -The fix: - -- `VaultSocketService.message` calls `relay.admitKnown` as its very first step: a WeakMap lookup by - socket object, the refused flag, a size compare and the bucket arithmetic. The first message of a - socket in a runtime parses the attachment once, then calls `admitRelay` (bucket keyed by socket id). -- Every message is charged its raw size: bytes for binary, UTF-16 units for text. -- The hard size cap is checked at the same point (binary 1,750,064 B, text 64 KiB; close 1009). -- On the first overdraft: `VAULT_BACKPRESSURE`, close 1013 `relay rate limit`, and the cumulative-ack - fence (`markFailed`): frames buffered before it still commit and ack, nothing later is acked. - Every later message of the socket is dropped in O(1) (`rawGateDrops`): no parse, decode, digest, - authority check or broadcast. A dropped sync update still counts in `updateFrames` and - `failedSocketDrops` (a 5-byte peek), so the outcome sum holds. -- The burst floor is now `MAX_DURABLE_UPDATE_BYTES + 64 + 64 KiB`, so one maximum raw frame plus its - envelope always fits. -- There is no earlier hook. After the upgrade, Cloudflare delivers WebSocket messages straight to the - Durable Object; they never pass through the Worker. Platform shedding (`Service overloaded`) of a deep - inbound queue cannot be prevented in the DO, only made less likely by cheap drops. -- Local (virtual time): 5 MiB/s of 64 KiB frames at the defaults (256 KiB/s, ~1.82 MB burst) closes the - flooder after 375 ms of input. About 29 frames (≈ the burst) are accepted with full work. Bystander - frames on the same body all commit, ack and broadcast. Not deployed. -- Still open: per-socket (not per-device) budget; a fresh burst on reconnect. Envelopes and replays are - now charged, so a reconnect resend backlog above the burst gets a 1013 and is resent later. - -## Test-only crash route - -`POST /vault/:id/debug/relay-crash` requires: - -- `YAOS_TEST_ONLY_DEBUG_ROUTES=true` -- an operator session -- `YAOS_RELAY_BODIES=true` and `YAOS_RELAY_GROUP_COMMIT` - -It drops every pending group buffer without committing or acking it, then swaps in a fresh runtime, -the same way `simulate-restart` does. The response is the `simulate-restart` JSON plus -`droppedRelayFrames`. diff --git a/docs/relay3-report.md b/docs/relay3-report.md deleted file mode 100644 index 6e3f0e96..00000000 --- a/docs/relay3-report.md +++ /dev/null @@ -1,159 +0,0 @@ -# Relay v3: cutting Cloudflare rows written — complete record - -Status as of 2026-10-02. Code: worktree `/Users/kavin/personal/obsidiansync/experiments/yaos-relay2`, branch -`relay-v2-spike`, HEAD `5656740` (21 commits on top of `3b66f1b`, where the v2 spike ended). Not pushed. -`yaos/` and `yaos-phase0/` untouched. Workers left deployed (reused `yaos-relay2-v1001-*` pool). - -Labels: **measured** = deployed Cloudflare run; **local** = local tests / row accounting; **inferred** = arithmetic -or model. Several fixes landed *after* the deployed run, so their numbers are local/inferred only (section 6). - -## 1. The problem - -Relay v2 (BODIES + LEAN + MB10) was faster than base but wrote ~5.9 Cloudflare rows per keystroke vs base ~3.1/edit -(measured, v2 C4). The Free plan caps 100k rows written/day; index entries, deletes and each `setAlarm()` count as -rows. The v2 cost model put relay lean at 53–73% of Free on a "typical day" (30 min of actual typing ≈ 1,800 words) -and strict mode over the cap. Causes: - -1. **One commit per keystroke.** v2 restored "durable before broadcast", so every frame was its own transaction. - The 10 ms micro-batch never caught two human keystrokes (~200 ms apart): mb10 5.54 vs mb0 5.55 rows/edit. -2. **~6 rows per commit**: journal row + index, body head, candidate receipt + index, plus checkpoint later - deleting each journal row and index entry again, plus catalog/alarm. - -Neither is inherent to relaying; both are design choices. - -## 2. Triage of the five reviews - -| Group | Ideas | Verdict | -|---|---|---| -| A. Cause | Commit per keystroke; 10 ms window useless; rows per commit; pay-twice deletes (review 2) | All agreed; adopted. Reviews 4/5 claimed "no fix without RAM buffering = data loss": half right — buffering is safe because the origin keeps edits until receipt and CRDT merge is idempotent | -| B1 | Share now, save in groups, receipt after save | **Done** | -| B2 | One tail row per note, cap 64 KB | **Done** | -| B3 | One receipt row per device (watermark) | **Done** | -| B4 | Fold indexes into primary key; head update per commit | **Done** | -| B5 | Client batches sends every 250 ms (review 3) | I recommended rejecting (adds latency); user asked for it; built, measured (1.53 rows/edit, +280 ms), then **removed entirely** | -| B extras | Save timing (300 ms idle / 1.5 s max / 64 KB); `setTimeout` not `setAlarm`; wake re-sync; client resend after 5 s | **Done** | -| B budget | Lengthen save window as daily budget runs low | **Dropped** (user: don't overengineer for Free) | -| C. Model realism | Typical day generous (1, 2) vs realistic (4, 5); plugins autosaving; account-wide limit; multi-device | Plugins measured; account-wide limit noted, not acted on | -| D. Imports | Bulk endpoint, snapshot imports, pack small notes, R2, budget-aware importer | **Out of scope here** — another agent owns the batch-import API | -| E. Fallbacks | Paid-plan note in docs; usage meter | Meter / "files written most today" deferred to later | - -## 3. Decisions made by the user - -- Drop "durable before broadcast" for **"durable before receipt"** (base worked this way before `e1ae3a4`). -- "Saved" confirmation may take 0.3–1.5 s instead of ~77 ms. -- No defensive code for DO death beyond: wake re-sync + client resend. -- Disk-originated changes batched 2 s settle / 5 s max. -- Skip budget-adaptive saving; "files written most" panel later. -- `vaultSync.ts` no-touch rule lifted; nobody else on this branch except the batch-import agent. -- Ship v3 once the revoked-device bug is fixed; keep 300 ms / 1.5 s / 64 KB. -- Remove B5 entirely. Fix R11 and HTTP-save cost without end-to-end re-runs. - -Checks that needed no work (client @ `a67aaca`): identical-content saves already send nothing (`diff.ts:71`, -`vaultSync.ts:2880`), blobs skip on hash match, `.obsidian/` and `.trash/` are always excluded (`exclude.ts`). - -## 4. What was built (commits on `relay-v2-spike`) - -**Server** (behind `YAOS_RELAY_GROUP_COMMIT=1`, needs BODIES + LEAN; v2 path unchanged for comparison): - -| Commit | Change | -|---|---| -| `28ed06b` | Group commit (broadcast immediately, buffer, commit on idle/max/bytes via `setTimeout`, receipts after commit), one tail row per body, one receipt row per device, schema trims, wake re-sync (step1 to every open socket), test route `debug/relay-crash` | -| `2141714` | Catalog alarm armed at most once per window (`YAOS_RELAY_GC_CATALOG_DELAY_MS`, 30 s); revoked frame no longer double-counted | -| `3cff026` | Group-commit test suite + `docs/relay3-group-commit.md` | -| `fd52578` | Cumulative-safe acks: socket fenced at a refused frame; acks held after wake until step2 durable; HTTP candidate/currentness flush the body buffer first | -| `d3b4db8` | **R11 fix**: authority writes (revoke/fence) first flush all buffered frames; authority cache invalidated at once; frames from a device revoked at flush are committed (they were already broadcast), socket closed 4403 | -| `14e5250` | HTTP candidate saves go through the group-commit store: 14 → 3 rows (no-op 1, replay 0) | -| `4a34f56` | **Commit-rate cap** `YAOS_RELAY_GC_MIN_INTERVAL_MS` = 1000 (idle commit also needs ≥ 1 s since last commit); **R12 raw rate gate**: bucket charged on raw message size as the first step, hard size cap, close 1013 on first overdraft, later messages dropped in O(1) | - -**Client** (`src/`): - -| Commit | Change | -|---|---| -| `21bc79d`, `de54b56` | Disk-originated markdown batching 350 ms / 2 s → 2 s / 5 s; typing unaffected | -| `7d70ea9` | Socket-ack receipts: cumulative confirmation, resend unconfirmed after 5 s (backoff to 60 s, ±20%), resend on reconnect, HTTP fallback after 15 s; settle without HTTP POST | -| `1a34693` | 7 VaultSync end-to-end-in-process receipt tests; status bar says "saving latest local state…" while waiting | -| `3aa7c44` | B5 removed from client, harness, deploy spec and cost model | -| `5656740` | Canvas identical re-save skips the `semantic//state` GET when the stored settlement already records this exact content at the current epoch (saves one DO request per identical re-save, incl. YAOS's own materialisation writes). Reset detection verified independent of that GET: socket `SEMANTIC_EPOCH_RESET_REQUIRED` fence, catalog-event feed on catch-up, fenced edits | - -**Harness / docs**: `21ff4a3`, `d067cd6`, `8730d32`, `e3a1b85` (v3 deploy spec, CRASH/FENCE/HTTPSAVE/autosave -scenarios, `costmodel3.py`); RFC `129c4eb`, `5f91664`, `ba57c63`. - -## 5. Deployed measurements (≈21:30–22:25 UTC, ~55 min) - -Taken at `e3a1b85`-era code: **before** R11 fix, HTTP-save change, B5 removal, commit-rate cap and rate gate. -Latency phases ran concurrently (`--lane-a-parallel`). - -| Metric | base | relay v2 | v3 | Label | -|---|---|---|---|---| -| Rows/keystroke, steady 5 keys/s (MB type5) | 5.58 | 5.63 | **0.29** (real client; 0.42 with candidateId) | measured | -| Typical day, % of Free | 30–52% | 51–53% | **3%** | inferred | -| Heavy day, % of Free | – | ~330% | 18% (assumed 5 keys/s; see 6) | inferred | -| Propagation p50/p99 (L2) | 299/696 ms | 63.0/396 | **47.7/76.3** | measured | -| L4 p99 (fast typing) | – | 2,305 ms | 231 | measured | -| Edit → "saved" p50/p99, real client | 479 ms (p50) | – | 353/1,177 ms; 0 HTTP POSTs | measured | -| Autosave 1/s × 8 h, open note | ~470% | – | 65% (2.15 rows/save) | measured per save, inferred per day | -| Autosave 1/s × 8 h, closed note (HTTP) | – | – | 82% (13.82 rows/POST) | same | -| Canvas identical re-save | – | – | 0 rows | client code | -| DO requests vs base | 1× | 1.15× | 1.15× typical, 1.34× heavy, ≤ 11% of Free | inferred | - -Correctness on v3 (measured): crash with buffered edits 10/10, 0 of 51 frames lost (incl. origin offline); -convergence 9/9, CW 121/121, reset races 20/20; B1, B6, B8, X1, X4, C5, C2-stress, HTTPSAVE pass; cumulative-ack -fence 5/5. **Failed:** B4 revoke convergence (R11 — fixed since, section 6); B7 flood 1/3 (R12 — one CF -`1013 Service overloaded`, one post-redeploy 1006, one pass; fixed locally since). - -## 6. After-run fixes (local tests only, not deployed) - -- **R11**: local repro of 10 buffered frames + revoke now converges; revoke racing broadcast converges; post-fence - frames never broadcast or stored. -- **HTTP save**: 14 → 3 rows/POST (≈ 4.0 incl. batched catalog event) → closed-note autosave day ≈ 24% of Free, - down from 82% (inferred by scaling the measured run). -- **Commit-rate cap** — the measured 0.29 was a best case; gaps > 300 ms committed per keystroke: - - | Typing | rows/key before → after | receipt p50 / max after | - |---|---|---| - | 1 key/s | 3.00 → 3.00 | 300 / 300 ms | - | 2 keys/s | 3.00 → 1.55 | 800 / 800 ms | - | 3 keys/s | 3.00 → 1.03 | 633 / 967 ms | - | 5 keys/s | 0.38 → 0.38 | 900 / 1,500 ms | - | 8 keys/s | 0.25 → 0.25 | 875 / 1,500 ms | - | bursty 0.1–2 s gaps | 2.61 → 2.23 | 300 / 1,362 ms | - - Bound: ≤ 1 commit/s per active note ≈ ≤ 3 rows per second of activity. RFC heavy day (2 keys/s, 8 h): - 115k–173k → ≈ 89k rows (inferred) — fits, narrowly. A 2 s min interval would give ≈ 45–50k at 2–3 s receipts. -- **Rate gate (R12)**: root cause was the limiter running after parse, SHA-256, authority check and a SQL read, and - counting only update bytes, so the DO fell behind and saw the overdraft only after the backlog. Local: 5 MiB/s - flood closed after 375 ms; 740 later messages dropped without parse/validate; bystander unaffected. -- Local suite: 187/187 suites, all three typechecks clean (at `ba57c63`). - -## 7. Invariants changed - -- "Durable before broadcast" → **"durable before receipt"**. A crash can lose buffered frames that peers already - applied; the origin was never acked so it resends, and wake re-sync pulls frames back from connected peers. -- Receipts are **cumulative** per socket; a refused frame fences the socket so no later ack covers it. -- Frames already broadcast are committed even if the device is revoked before flush; nothing after the fence is - broadcast or stored. - -## 8. Open items - -1. **Deployed re-check** (~20 min) of everything in section 6: C4/MB rows at several typing rhythms, B4 revoke, - B7 flood (serial), HTTPSAVE. Nothing after `e3a1b85` has run on Cloudflare. -2. Heavy-day margin ≈ 89% with a 1 s min interval; decide whether to raise it to 2 s. -3. Rate gate: ~1.8 MB of flood still fully processed before cutoff; CF can still shed a saturated DO; budget is - per socket and resets on reconnect; verify a large reconnect resend (merged into one message) never exceeds the - burst/size cap and gets refused forever. -4. R10: DO requests 1.15× base (bar 1.1×) still needs sign-off. -5. Not re-run on v3: C1 CPU, X1 at scale, R2 starvation, stored-state growth. -6. HTTP fallback replay still writes a 4-row idempotency receipt path in rare cases (after 15 s without receipt). -7. Existing gap (not caused by v3): a semantic reset sends nothing to root sockets, so a connected client with that - canvas closed and otherwise idle notices only at its next catch-up or edit; writes stay fenced, no data risk. -8. Imports (batch-import API) belong to the other agent; "files written most today" panel deferred. - -## 9. Files - -- This report: `experiments/results/relay3/REPORT.md` (copy in repo: `docs/relay3-report.md`) -- Deployed tables: `experiments/results/relay3/RESULTS.md` -- Server design, row accounting, every fix: `experiments/results/relay3/SERVER-NOTES.md` -- Client changes, timing audit, Canvas/open-note findings: `experiments/results/relay3/CLIENT-NOTES.md` -- RFC: `yaos-relay2/docs/rfc-relay-bodies.md` (copy `experiments/results/relay3/RFC-relay-bodies.md`) -- Group-commit design: `yaos-relay2/docs/relay3-group-commit.md` -- v2 baseline: `experiments/results/relay2/REPORT.md`, `RESULTS.md` diff --git a/docs/rfc-relay-bodies.md b/docs/rfc-relay-bodies.md deleted file mode 100644 index d379aaec..00000000 --- a/docs/rfc-relay-bodies.md +++ /dev/null @@ -1,1344 +0,0 @@ -# RFC: Relay Markdown body updates - -Status: **final** (relay v2 spike, branch `relay-v2-spike`, base `5dd32f3`), 2026-10-01. All measured values come -from the full run, tag **f1001** (raw output: `experiments/logs/relay2/runall-f1001/raw/.json`; tables in -`experiments/results/relay2/RESULTS.md`, report in `REPORT.md`). Each cell names its scenario id and n. - -What this RFC describes: -- The server implementation as committed through `7feae39` (round 4) on 2026-10-01. Earlier rounds: `20e0e34`/`e1abb36` - (round 1), `54cee51` (round 2), `4cdc37e` (round 3). Harness: `35b2882`, `276c16c`, `66fb78c`. Nothing described - here is uncommitted. The earlier "(WIP)" marks are gone. The full run used harness SHAs `d929410` (lane B, fresh - `yaos-relay2-f1001-*` workers), `45f3705` (reused phases) and `35e8b06`; the server code is identical in all three. -- Full-run configurations (`scripts/relay2/runall.sh`): - - **relay** (the v2 candidate) = `YAOS_RELAY_BODIES=true` + `YAOS_RELAY_LEAN_ROWS=true` + `YAOS_RELAY_MICROBATCH_MS=10` - - **relay-strict** = relay on, lean off, mb 0. This is the per-frame transaction variant, run for L2, L4, C1, C2, - C4, B7, X2 and the MB sweep. - - **base** = flag off, same vars (inert) - - All three set `YAOS_RELAY_RESET_COOLDOWN_MS=0`. -- Cited line numbers drift. Function names are the stable reference. -- The contract document is [`relay2-protocol.md`](relay2-protocol.md). Where the two differ, the code wins, and the - difference is called out. - -**Relay v3 addendum (2026-10-01, runs r3-1002/b/c).** v3 is group commit behind `YAOS_RELAY_GROUP_COMMIT=1` -(sections 4.4a, 6, 8.8, 9.1, 12 R11–R14, 13). It was measured on deployed reused `yaos-relay2-v1001-*` workers with -harness `d067cd6`/`8730d32`/`e3a1b85` and server code through `fd52578`. Raw data: -`experiments/logs/relay2/runall-r3-1002{,b,c}/raw/`. Tables: `experiments/results/relay3/RESULTS.md`; report: -`REPORT.md`. Latency phases in r3 ran concurrently (`--lane-a-parallel`) and are labelled **[conc]**. - -Evidence labels: -- **measured**: taken on a deployed Worker or in a local run, with n and the scenario ID. -- **inferred**: derived from measurements through a stated model. -- **code**: read from `server/src`. -- **smoke**: small n (v0930 = `8e194a4` default rows; w0930 = `276c16c` primary configs). Directional only. - -Full-run concurrency labels (RESULTS.md run index): -- **[serial]**: L1, L2 ×3, L3 ×2, L4-strict. -- **[conc]**: run concurrently, each on its own worker/DO (user-approved): L4-base/relay, L5 ×4, L6 ×2, L7 ×2. -- **[B∥10]**: lane B, non-latency phases, 10 in parallel (C1, C2, C4, C5, X1, X3, X4, K1, MB). -- **[serial-after]**: B7 ×3 and X2 ×3, after everything else. - -## 1. Summary and recommendation - -**Recommendation: GO-WITH-CONDITIONS.** Ship relay bodies behind `YAOS_RELAY_BODIES`, configured as lean rows + -mb 10. Basis: full run f1001, L1–L7, C1–C6, B1–B8, CW, X1–X4, K1 deployed at full n, plus K2/K3 local (section 8). - -Conditions: -1. **Lean rows are mandatory.** Strict (lean off, mb 0) is 107–119% of the Free rows/day budget on a typical day - (inferred, costmodel runs B/D, section 9). Relay lean + mb 10 is 53–73%. -2. **Fix or explain the relay p99 tails before default-on.** L4-relay p99 was 2,305 ms (measured, n = 4,990, [conc]), - C2-quick-relay propagation p99 9,132 ms (measured, n = 5,000, max-rate, [B∥10]), and the checkpoint alarm reached - p99 1.04 s CPU on a 50k-edit body (measured, C2-stress-relay). Cause open; the likely suspect is the alarm. -3. Socket-ack receipts in `VaultSync` (section 11, phase 2). -4. Decide reset starvation (G9/R2) before reset ships. -5. Add receiving-side validation (R7). - -Basis (all measured, full n): -- **Propagation:** 4.67× faster at p50 (L2 308 → 66.0 ms, n = 290, [serial]) with a better p99 (608 → 314 ms). - Reversal condition 1 is not triggered. -- **Receipts:** edit → cleared receipt 6.2× faster (L5b1 prod 480 → 77.0 ms p50, n = 90, [conc]). -- **Sockets:** no 32-socket cap (X1-relay 2,000/2,000, highest step tested; base 429 at #33). -- **Hibernation:** the relay socket survives idle and 2 restarts; base loses all 4 sockets with 1008 (B1). -- **Flood control:** our 1013 at 26.9 s with the bystander −1.3% (B7-relay). -- **CPU:** relay WS CPU p99 4.6 ms vs base 30.7 ms (C2-quick, n = 5,000). Base already exceeds the 10 ms bar. -- **Correctness:** relay convergence 9/9 (B5 20/20 races), strict 3/3, and the frame-accounting identity holds in - 16/16 relay/strict runs. - -Design notes that still apply: -1. Semantic reset moves to clients under a lease. That is a prerequisite for large notes, because only clients can - compact past the 96 MiB Wasm cap. -2. ywasm stays on the server, for the root document, Canvas semantic documents and the base fallback. It leaves the - Markdown body hot path. -3. Reversal conditions are listed in section 13 with their full-n status. None of the latency, rows, correctness, - CPU-per-message, memory or scale conditions is triggered. Condition 3 (DO requests) is exceeded on the letter by - the model (typical day 1.15× base, both at 2–3% of Free) and needs explicit sign-off; condition 7's starvation - clause was not measured. - -### 1.1 Relay v3 addendum (group commit) - -**Recommendation: adopt v3 as the relay write path (GO-WITH-CONDITIONS), with B5 off.** It replaces "lean rows + -mb 10" as the shipping config. - -Basis (measured unless labelled): -- **Rows.** The real client (no `candidateId`) writes **0.29 rows/edit** at 5 keys/s (gql, MB-v3nc-type5, n = 551), - against v2's 5.63 in the same run (**−95%**). That is 1.45 rows per typing-second. Spaced edits cost 2.0 rows - each (C4 counter). -- **Free rows** (inferred, section 9.1): typical day **3%** (v2 51–53%, base 30–52%), heavy 8 h day 18%. 100k rows - needs 19 h of non-stop typing. A 1 s plugin autosave for 8 h adds 62k rows (open note) or 80k (closed note). -- **Latency.** Propagation is unchanged or better: L2 p50 / p99 47.7 / 76.3 ms vs v2 63.0 / 396 (n = 290, [conc]). - L4 p99 is 231 ms (f1001 v2 2,305). The receipt moves to 0.35 s p50 for a lone edit and 0.9 / 1.6 s p50 / p99 - while typing continuously (user-approved 0.3–1.5 s window plus RTT). L5 real `VaultSync` edit → cleared is 353 ms - p50, with 0 HTTP POSTs (base prod 479). -- **Crash with a pending buffer:** 10/10 rounds, 0 frames lost, including recovery from the peer when the origin is - offline. -- **Correctness:** convergence suite 9/9. Cumulative-ack fence: too-large 5/5, rate 6/6 rounds that drew a - refusal. B1, B5 20/20, B6, B8, CW 121/121 and X4 pass. - -Conditions (v3): -1. **R11: fixed locally (d3b4db8), not re-measured on a deployment.** B4-v3 failed convergence because a revoked - device's pre-revocation frames reached peers and were dropped at flush. Now the revocation and authority-fence - paths flush every group-commit buffer in the same DO turn, before the authority write. A frame admitted after - the fence is refused (4403) before any broadcast. Re-run B4-v3 before default-on. -2. **R12: fixed locally (4a34f56), not re-measured on a deployment.** B7-v3 passed 1 of 3. In one attempt - Cloudflare shed the bystander with `1013 Service overloaded` before our rate limit closed the flooder. The - bucket was checked late (after decode, digest and a SQL read) and charged update bytes only. Now every message - is charged its raw size as the first step of the DO message handler, and later messages of a refused socket - are dropped in O(1). Locally (virtual time) the 5 MiB/s flooder is closed after 375 ms of input. Re-run B7-v3 - serially before default-on. -3. **B5 send-coalescing was measured and removed (R13).** At idle = 300 ms it gave 1.53 rows/keystroke vs 0.29 - (v3nc) and added about 280 ms to propagation, so the client has no coalescing knob. Parameters stay at - 300 ms idle / 1.5 s max / 64 KB, plus a commit-rate cap (4a34f56, `YAOS_RELAY_GC_MIN_INTERVAL_MS` = 1000): - an idle commit waits until 1 s after the body's previous commit. Without it, typing at 2–3 keys/s committed - per keystroke (3.00 rows/key, local accounting); with it 1.55 / 1.03 (inferred, section 9.1 note). -4. v2 conditions 3–5 still apply (receipts in `VaultSync`, which are now landed and measured in L5; reset - starvation; receiving-side validation). v2 condition 1 (lean rows mandatory) is **retired** by v3. v3 requires - lean rows by construction, and rows are no longer near the cap. v2 condition 2 (p99 tails) is **largely - answered** for the hot path (L2 p99 76 ms, L4 p99 231 ms, [conc]). The checkpoint-alarm CPU part of R9 was not - re-measured. - -## 2. Problem - -Today the authoritative body path puts the following on every body frame: -- a resident ywasm document, plus a validation mirror document -- a debounced flush -- a second write path: the HTTP candidate, which carries the same bytes - -The costs below were measured before this spike. - -| Cost | Evidence | Label | -|---|---|---| -| Keystroke propagation p50 ≈ 300 ms, set by the 250 ms `PERSIST_DEBOUNCE_MS` flush plus commit and broadcast | A3: 306 ms propagation, 301 ms body sync. v1 relay L2: 300 → 52 ms (A5) | measured | -| Idle storm before phase 0: ≈160k billed DO requests/day for 3 idle devices, 1.6× the DO free tier | A1: 1,430 reconnects per socket per day; `1008 socket authority mismatch` on every ping after a wake | measured + inferred | -| Body sockets capped at 32 per vault (`MAX_BODY_SOCKETS`, the 48 MiB / 96 MiB Wasm envelope) | A3: the 33rd socket gets 429. Harness X1 (base): 429 `body_socket_limit` at #33 | measured | -| Resident memory grows with open bodies: 32 × 512 KiB adds 36 MB of Wasm on base; flat on relay | A5 C3 | measured | -| CPU per keystroke on a heavy note: 19 ms base against < 1 ms v1 relay | A5 C1 | measured | -| Hibernation: base closes body sockets with 1008 on the first edit after DO eviction. Reconnect ≈ 1.33 s against 351 ms warm | A5; harness B1 (base, scratch-2) | measured | -| Time from edit to cleared settled receipt p50 is 531 ms. The debounce accounts for 250 ms of it and the candidate POST for ≈ 280 ms | L5 baseline, D8, n = 14, scratch-2 | measured (small n) | -| Every Worker → DO hop costs ≈ 44–47 ms; a HEAD is 4 hops at ≈ 215 ms | A2/A4 | measured | - -Baseline tables for this spike, flag off, same code and harness (measured, full run f1001, 2026-10-01, base `5dd32f3` -+ spike flag off; deploy version ids in section 14.2): - -| ID | Metric | Base p50 / p90 / p99 (ms) | n | Date (UTC) | Worker | -|---|---|---|---|---|---| -| L1 | ping RTT | 50.0 / 53.9 / 247 [serial] | 90 | 2026-10-01 15:12 | `yaos-relay2-v1001-c1-relay` (reused, fresh vault) | -| L2 | propagation A→B, 4 KiB | 308 / 334 / 608 [serial] (small n was 412 p50, n = 30) | 290 | 2026-10-01 15:13 | `yaos-relay2-v1001-c1-strict` (reused) | -| L3 | propagation, 64k chars, before / after the quick trace | before 304 / 322 / 614; after 340 / 349 / 356; trace replay per frame 261 / 366 / 597 [serial] | 40 + 40 (+ 5,000) | 2026-10-01 15:22 | `yaos-relay2-v1001-c2-stress-base` (reused) | -| L4 | per-frame at 25 edits/s | 213 / 313 / 543 [conc] (small n was 293 / 458) | 4,990 | 2026-10-01 15:29 | `yaos-relay2-v1001-l1-relay` (reused) | -| L5 | edit → cleared receipt, 1-edit burst | prod 480 / 553; nodebounce 237 / 326 [conc]. 8-edit burst first / last: prod 1,356 / 475, nodebounce 1,165 / 280 | 90 per mode | 2026-10-01 15:29 | `yaos-relay2-v1001-c5-{bursts,catchups}-base` (reused) | -| C1 | propagation small / heavy; WS CPU per invocation | 303 / 385 ms p50; WS CPU p50 / p90 / p99 1.36 / 2.27 / **64.3 ms** (5,101 invocations, gql) [B∥10] | 40 each | 2026-10-01 13:48 | `yaos-relay2-f1001-c1-base` | - -Reused workers: 40 phases ran on idle `yaos-relay2-v1001-*` workers with fresh vaults because of the account's -500-DO-namespace cap (test-environment note, section 12). Worker names on reused workers are historical; the -variant comes from the deployed vars, recorded per run. - -The base suites pass on the base SHA (measured, `base-tests.md`): -- `test:regressions`: 179/179 suites, 3,135 assertions, 59 s -- node-runtime: 32/32 -- conformance: 26/26, with 1 declared GAP -- headless: 92/92, with 1 of 4 runs flaky ("rename retires the old active path") -- pack-smoke fails under `npm run` because of an environment leak. This is pre-existing. - -## 3. Product constraints and how they shaped the design - -| Constraint (brief §1.2) | Consequence in this design | -|---|---| -| **Local-first; disk is the user-visible truth; the server is not the only copy.** | The server can be a durable ordered mailbox, not a judge. It stores and orders bytes, fences on authority and epoch, and relays. Validation of body content moves to the receiving client before its disk write (section 5), because that client owns the disk. Losing the server's semantic view is acceptable only because every device holds the full text. | -| **Few clients × many documents** (1–5 devices; hundreds to tens of thousands of notes). | Per-note server state must be cheap and bounded by bytes, not documents. Relay sockets carry no resident document. The only cache is a byte LRU (`YAOS_RELAY_MERGED_CACHE_BYTES`, 16 MiB) keyed by head sequence. The socket cap is a config value (default 5,000), not a memory envelope. Bootstrap and catch-up read stored bytes and never build a document. | -| **Idle most of the day, Obsidian left open.** | Hibernation must be invisible. Relay sockets are exempt from the `runtimeEpoch` fence (the attachment records the admission epoch), so a woken DO keeps serving the socket (B1). No debounce timers are needed. An alarm is armed only when a tail crosses the checkpoint threshold. A fully idle vault does no SQL writes. | -| **Self-hosted on the user's Cloudflare account, aiming for $0.** | The free tier is a hard constraint on request counts and rows written (section 9). It drove these choices:
• one transaction per append, with no extra rows (`vault_operation_outcomes` is not written)
• the growth cap, so resends become no-ops rather than rows
• micro-batching (0..250 ms; recommended 5–10)
• lean rows: no clock write, inline attribution and hash, coalesced catalog events. That is 3 CF rows per plain append instead of 9.
• checkpoints bounded to 200 rows per pass
It also exposed the main risk: the relay writes one append per frame (or per micro-batch), where base writes one per 250 ms window (section 12, R5). | -| **Devices may all be phones; nothing may require a desktop.** | Server-side semantic compaction is replaced by a client lease, so the work must be feasible on a phone.
• K2 measured desktop reset build times of 12–98 ms p50 (n = 10).
• Mobile-class ×4–6 is inferred at 48–586 ms, with a duty-cycle pessimistic bound of 1.7 s p50 for 5 MB.
• The routine checkpoint stays on the server as a byte merge that needs no client.
• Any device may take the lease; none is privileged. | -| **Devices are usually close to the DO; mobile networks and travel exist.** | Receipts ride the socket ack, so there is no second HTTP round trip. Catch-up and bootstrap stay HTTP-batched. Retries are idempotent through `candidateId` receipts. The lease TTL (120 s, up to 600 s) and the binary reset upload reflect slow mobile uploads: a pre-round-2 JSON upload of 5 MB took 115.7 s (B5). After round 2, a binary 5.24 MB reset takes 1.7–3.3 s from a desktop (core3, n = 4). | - -## 4. Design - -### 4.1 Scope and flag - -- The flag is `YAOS_RELAY_BODIES === "true"` (`relayFlag.ts` `relayBodiesEnabled`). With it on, Markdown body sockets - use the relay path. A body is a relay body when it is not `root` and has a catalog head (`server.ts` `isRelayBody`). -- Out of scope: - - root sockets (download-only) - - Canvas semantic sockets (`/ws/semantic`) - - lifecycle HTTP routes, i.e. create, rename and delete, including the creation-candidate fence -- The Node host reads the same flag (`packages/server-node/src/index.ts`). -- The capability `relayBodies: 2` is advertised in `VAULT_READY.capabilities` and in `/api/capabilities`. -- The socket URL does not change: `/ws/body/:bodyId`, with a ticket bound to the body epoch. - -### 4.2 Data model - -The relay reuses the storage-4 tables. It adds two lazily created tables. Lean mode also adds nullable columns to -`vault_journal`. - -| Table | Relay use | -|---|---| -| `vault_clock` (single row) | Default: one `sequence + 1` per append. This is the vault-wide total order. Lean: not written per append. The sequence is `MAX(vault_clock, MAX(journal.sequence)) + 1`, and `syncLeanClock` raises the clock before any journal delete, so a sequence is never reused. | -| `vault_journal` (`sequence`, `document_id`, `generation`, `semantic_epoch`, `kind`, `update_byte_length`, `data`, `created_at`; index `(document_id, sequence)`) | One row per append, kind `'body'`, raw Yjs v1 update bytes of at most 1,750,000 B (`MAX_DURABLE_UPDATE_BYTES`). A reset writes kind `'semantic-reset'` with zero bytes. Lean adds nullable `attr_*` columns (frame-0 attribution) and `relay_content_hash`/`relay_size` (the accepted hash), via a lazy `ALTER TABLE` in `enableLeanRows`. | -| `vault_mutation_attribution` | Default: one row per frame in the append, holding principal, membership revision, device, credential revision and `request_digest`. A candidate frame carries `operation_id = candidateId` and `request_digest = candidateDigest` (G14, round 3), so `/operations/:id/outcome` resolves. Lean: frame 0 is inline on the journal row, and only frames 1..n of a micro-batch get rows. | -| `vault_document_heads` | `generation + 1` and `latest_sequence` per append. Epoch changes happen only through reset. | -| `vault_catalog_events` | Default: one row per append, holding path, lifecycle `active`, generation, epoch, and `content_hash`/`size` (NULL when no claim was accepted). Lean: written by the delayed relay alarm, one event per dirty body per pass (`coalesceLeanCatalog`). Readers overlay the newest journal row (catalog-head overlay). | -| `vault_candidate_receipts` | One row per enveloped frame that carries a `candidateId`: `(body_id, client_id, candidate_id)` → digest, epoch, generation, sequence, runtime epoch. Pruned by the alarm (`pruneCandidateReceipts`). Kept in lean mode. | -| `vault_checkpoints` + `vault_checkpoint_manifests` | Byte-merged snapshot, chunked. Written by the checkpoint and by reset. The 3 newest checkpoints of the current epoch are kept, and pins are respected (`pruneUnpinnedDocumentHistory`). | -| `vault_semantic_compaction_state` | Reset cooldown source (`last_compacted_at`). | -| **`relay_compaction_leases`** (new) | `body_id` PK, `lease_id`, `device_id`, `principal_id`, `body_epoch`, `expires_at`, `created_at`. Created lazily by `RelayBodyStore.ensureLeaseTable`. Rows are deleted on revocation and fence (G19). | -| **`relay_body_budget`** (new, round 3) | Persisted over-budget marker per body (G20). It survives eviction, so the alarm does not retry a merge that is known to be too large. Keyed by epoch, so a reset (new epoch) clears it. | - -Nothing is migrated. With the flag off the lease table is never created. With the flag on, base tables are written -with the same meanings as base, so a vault can move between modes only on a fresh install (section 11). - -### 4.3 Wire protocol - -**Frames on a relay body socket** (`vaultSocketService.ts` `relayMessage`): - -| Direction | Frame | Handling | -|---|---|---| -| C→S text | `BODY_UPDATE_ENVELOPE` JSON: `clientFrameId`, `payloadDigest` (sha256 hex of the next binary update), optional `candidateId`, `candidateDigest`, `contentHash`, `size`, `stateVector` (base64) | Held in `pendingEnvelopes[socketId]` (memory only) and paired with the next binary SYNC_UPDATE on that socket when the digest matches (`relayBodies.ts` `handleControl`, `handleSyncFrame`). A mismatch drops the pairing (`envelopeMismatches`). The frame is still appended, unenveloped. | -| C→S binary | y-protocols `MESSAGE_SYNC` step1 / step2 / update | step1 → step2 served from bytes (4.5). step2/update → append (4.4). | -| C→S binary | awareness | Relayed through `relayAwareness`; not stored. | -| C→S text | `VAULT_PING`, `BODY_CURRENTNESS_QUERY` | Both re-validate the actor. The ping answers with the admission `runtimeEpoch`. Currentness answers from `currentBodyHead` (the hash may be null). | -| S→C text | `VAULT_READY` | Includes `capabilities.relayBodies: 2`. | -| S→C text | `BODY_COMMITTED` (origin) | `relay: true`, `clientFrameId`, `vaultSequence`, `durableGeneration`, `candidateId`/`candidateDigest`, `payloadDigest`, `runtimeEpoch` (admission), `commitRuntimeEpoch`, `contentHashAccepted`, `deduped`, `noop`. | -| S→C binary + text | peer fan-out | Raw SYNC_UPDATE with the same bytes, plus `BODY_COMMITTED` with `relay: true, peer: true`. The origin is excluded. | -| S→C text | `BODY_UPDATE_REJECTED` | Reason `candidate_id_reused`: the same `candidateId` was sent with a different digest. | -| S→C text | `VAULT_BACKPRESSURE` | Reason `relay_rate_limit`, followed by close 1013. | -| S→C text | `VAULT_ERROR` | Code `durability_failed`, followed by close 1011. Code `relay_frame_error` (round 4), followed by close 1011. | - -**Close codes:** - -| Code | Meaning | -|---|---| -| 4409 | Epoch fence. Carries the current epoch; the client rebases. | -| 4403 | Authority superseded (revocation, membership change). Since round 3 this also covers step1 (G6) and micro-batch flush (G2). | -| 1013 | Our rate limit: reason `relay rate limit`, always preceded by `VAULT_BACKPRESSURE` (`rateLimitCloses`). Base also sends 1013 with `semantic compaction pressure`, `body cache budget exceeded` or `pending durability budget exceeded`. **Reason `Service overloaded` or an empty reason is the Cloudflare platform** shedding an overloaded DO. The server never sees those closes and no counter moves. The transient 1013 seen in K1 (mb 0 flood) was the platform. | -| 1009 | Frame larger than 1.75 MB. | -| 1008 | Body not active or deleted. | -| 1011 | Commit failed (`commitFailures`), or **relay frame error** (round 4): any throw before the append, on the socket path or the micro-batch timer path (`frameErrors`, `lastFrameError`). Before round 4 a throw in `relayMessage` sent `VAULT_ERROR` and kept the socket open, which silently lost the frame. | -| HTTP 429 | At upgrade, when relay + semantic sockets would exceed `YAOS_RELAY_MAX_BODY_SOCKETS`. | - -**No silent drops (round 4).** Every non-empty update frame has exactly one counted outcome. Diagnostics expose the -identity: - -``` -updateFrames = appendFrames + noopSkips + dedupeHits + dedupeConflicts + batchDuplicateCandidates - + authorityCloses + authorityDrops + rateLimitCloses + epochFences + bodyInactiveCloses - + tooLargeCloses + commitFailures + frameErrors -``` - -A random-interleaving unit test checks it at mb 0 and mb > 0. Throws after commit are counted (`postCommitErrors`) and -never skip a peer's fan-out. The harness checks the identity on every full-n run's diagnostics delta. - -**HTTP routes added:** -- `POST /vault/:id/body/:bodyId/compaction-lease` and `POST …/semantic-reset`. Both need the - `vault.content.write` capability. See 4.9. -- `HEAD /body/:id`. -- Test-only: `/__yaos/test-only/relay-table-counts`. - -`GET /body/:id` (and HEAD) gain `x-yaos-head-sequence` and `x-yaos-content-hash-state` (`known` | `materialised` | -`unknown`). - -### 4.4 The append - -Order of operations, as implemented through round 4. Code: `relayBodies.ts` `handleSyncFrame` then `commitFrames`, -and `relayBodyStore.ts` `appendRelayBodyUpdate`. - -1. **Size.** More than 1,750,000 B closes the socket with 1009 (`tooLargeCloses`). -2. **Envelope pairing.** `sha256HexSync(update) === pending.payloadDigest`. A mismatch counts in `envelopeMismatches`, - and the frame is treated as unenveloped. -3. **Empty skip.** `[0,0]` or zero bytes: no append (`emptySkips`, outside `updateFrames`). An enveloped frame gets a - `noop` ack. -4. **Authority** (before dedupe since round 3, G7). `validateActorCached`. The cache is keyed by the full actor tuple - and invalidated by `authorityVersion`, which every in-process authority writer bumps (revoke, membership, authority - fence). The TTL is 5 s. Failure closes with 4403 (`authorityCloses`). -5. **Candidate dedupe.** - - Look up `vault_candidate_receipts(body, device, candidateId)`. - - Same digest: re-ack with the stored receipt (`deduped: true`, `dedupeHits`). - - Different digest: send `BODY_UPDATE_REJECTED` (`dedupeConflicts`). -6. **Budget.** A per-socket token bucket: - - 256 KiB/s - - burst `max(YAOS_RELAY_BURST_BYTES, 1.75 MB)`, so that any legal frame can pass (round 2) - - empty bucket: `VAULT_BACKPRESSURE`, then 1013 (`rateLimitCloses`) -7. **Optional micro-batch** (`YAOS_RELAY_MICROBATCH_MS`, default 0, clamped to 0..250; it was 0..50 before round 3). - - Frames are queued per **(bodyId, epoch)** (G3), and the batch is merged with `mergeUpdateBytes` at commit. - - At flush, every frame's actor is re-validated (G2). A revoked frame is dropped and its socket gets 4403 once - (`authorityDrops`). - - A duplicate `(device, candidateId)` inside a batch (G11) is re-acked if the digest is the same, or rejected if it - differs (`batchDuplicateCandidates`). - - `semantic-reset` flushes the body's pending batches before its CAS. - - Recommended: mb 5–10, which is the full-run relay config (mb 10). -8. **Epoch check** against the head, per batch. The merged cache revalidates against the head row. A mismatch fences - the batch's frames with 4409 (`epochFences`). -9. **Growth cap.** - - Merged body ≤ 256 KiB (`EXACT_MERGE_BYTES`): exact merge. The append is a no-op if and only if - `mergeUpdates([merged, update])` is byte-equal to `merged`. The SV is exact. - - Larger bodies: the append is a no-op only if the update is byte-identical to the last update (≤ 64 KiB kept). - The next SV is the pointwise max of the head SV and the update's SV, flagged `stateVectorExact = false` (G8). - - A no-op is acked `noop: true` and writes no rows (`noopSkips`). -10. **Currentness (D6 #7).** The newest paired envelope whose `stateVector` equals the next merged SV contributes - `contentHash`/`size`, but only if the SV is exact (G8, round 3: a body over 256 KiB never accepts a claim). - Otherwise the hash is NULL. -11. **One `transactionSync`**, `appendRelayBodyUpdate`: - - read `vault_document_heads`: exists, epoch == expected - - read the latest `vault_catalog_events` (lean: plus the overlay): `active`, `file_id` matches, no - `vault_creation_candidates` row - - default: `UPDATE vault_clock SET sequence = sequence + 1 RETURNING sequence`. Lean: `MAX(clock, journal max) + 1`, - with no write. - - `INSERT vault_journal (…, 'body', bytes)`. Lean also writes the inline `attr_*` and `relay_content_hash`/`relay_size`. - - attribution: N × `INSERT vault_mutation_attribution` by default, frames 1..n only in lean - - `UPDATE vault_document_heads SET generation = generation + 1, latest_sequence = seq` - - catalog: `INSERT vault_catalog_events (…, content_hash|NULL, size|NULL)` by default. Lean defers it to the - coalescing alarm. - - 0–N × `INSERT vault_candidate_receipts … ON CONFLICT DO NOTHING` - - **Measured CF rowsWritten per append** (index writes included; `scripts/relay2-core-smoke.ts`, n = 20 acked edits - per frame kind; diagnostics `rowsWritten` delta): - - | Mode | Plain frame | Candidate frame | Worker | - |---|---:|---:|---| - | default | 9 | 11 | `yaos-relay2-coresmoke-1` | - | lean | 3 | 5 | `yaos-relay2-corelean-1` | - - Local node rows, amortised over a checkpoint pass (`relay2-core-alarmprobe.ts 750 30`): lean 2.13 against default - 5.07 per append (measured, local). The lean coalescing pass writes 3 rows per dirty body per pass. -12. **After commit only**, in this order: - 1. update the merged cache entry - 2. `syncDocumentCache`. Since round 3 (G5) it never applies the update to a resident base doc. It discards a clean - resident; a dirty resident is left stale and counted (`residentStaleSkips`). - 3. origin `BODY_COMMITTED` - 4. peer fan-out of the same bytes, at the same epoch, excluding **every** origin of the batch (G16) - 5. `notifyBodyCommitted` - 6. arm the checkpoint alarm if the tail is ≥ 50 rows or ≥ 1 MiB. Lean arms it on every append, delayed by - `YAOS_RELAY_LEAN_CATALOG_DELAY_MS` (2 s). - - A throw in any of these steps is counted (`postCommitErrors`) and the remaining steps still run. -13. **On a commit exception:** `VAULT_ERROR durability_failed`, close 1011 (`commitFailures`), invalidate the cache. A - throw anywhere before the append closes 1011 too (`frameErrors`). The client resends on reconnect, and the growth - cap or the receipt deduplicates the resend. - -The default relay writes the same row set that base writes per debounced flush (flag-tests.md). What changes is how -often: once per frame or micro-batch, instead of once per 250 ms window per body. Lean mode removes the clock write, -the frame-0 attribution row and the per-append catalog event. - -### 4.4a Relay v3: group commit (`YAOS_RELAY_GROUP_COMMIT=1`) - -Full design: [`relay3-group-commit.md`](relay3-group-commit.md) (code through `fd52578`). v3 needs -`YAOS_RELAY_BODIES=true` and `YAOS_RELAY_LEAN_ROWS=true`. With the flag off, the schema and every path are v2. - -- **Share now, save in groups, confirm after saving.** Every message is first charged to the socket's rate budget - on its raw size (R12). Admission is then as in v2: size, envelope, authority and dedupe. The frame is - **broadcast at once** and buffered per (body, epoch). The buffer commits in one `transactionSync` after 300 ms - idle (`YAOS_RELAY_GC_IDLE_MS`) and at least 1 s after the body's previous commit (`_MIN_INTERVAL_MS`), 1.5 s - after the first frame (`_MAX_MS`), or at 64 KB (`_MAX_BYTES`, synchronous). HTTP reads, the authority fence - and semantic resets flush at once. At flush the checks run again: G2 authority, epoch, B6 dedupe, in-buffer - duplicates and the growth cap. Receipts (`BODY_COMMITTED`, v2 wire format) go out after the commit returns. -- **Rows.** One `relay_body_tail` row per body (WITHOUT ROWID, records since the last checkpoint), one - `relay_device_receipts` row per device (a 256-entry JSON ring, 15 min TTL), and the head update. No journal row, - no attribution rows, no per-commit `setAlarm` (the catalog alarm is armed at most once per 30 s). - A tail checkpoint (≈ 12 rows) runs every 64 KB or 512 records. -- **Cumulative acks and fences** (invariant 3): a refused frame fences its socket, so an ack never passes a refused - frame. -- **Wake re-sync** (invariant 1): a new runtime sends step1 with the durable SV to every surviving relay socket. - Each client's step2 returns frames that a crashed runtime broadcast but never committed. A covered step2 writes - nothing. -- **Client resend.** `VaultSync`'s `RelayReceiptChannel` (`relayReceipts`) resends a frame that has no receipt after - 5 s, and on reconnect. The real client sends no `candidateId` on relay-covered candidates. It confirms them with a - synthesized receipt and falls back to HTTP at 15 s. So the realistic v3 client is the `relay-nocand` adapter (spec - `v3nc`), which writes no receipt row. -- **B5 client send-coalescing** (250 ms, measured as `v3b5`, since removed from client and harness): the client merged - updates for up to 250 ms before sending. - -**Rows per edit and per commit** (measured, relay3 run, deployed `yaos-relay2-v1001-*` workers, harness counters = -`rowsWritten` diagnostics delta; gql values in section 8.8): - -| Config | C4 rows/edit (1 edit per 1.5 s, n = 50; every edit is its own commit) | MB type5 rows/edit, counter / **gql** (5 keys/s, 110 s, n = 551) | Commits/s while typing | Rows per commit | -|---|---:|---:|---:|---:| -| base | n/a | n/a / 5.58 | n/a | n/a | -| relay v2 (lean, mb 10) | 5.0 | 5.00 / 5.63 | n/a (1 append per frame) | n/a | -| v3, harness client with `candidateId` | 3.0 | 0.38 / 0.42 | 0.64 | 3 (tail + head + receipt ring) | -| **v3, real client (no `candidateId`, `v3nc`)** | **2.0** | **0.25 / 0.29** | 0.64 | 2 (tail + head) | -| v3 + B5 250 ms | 3.0 | 1.50 / 1.53 | 2.5 | 3 | - -While typing continuously, the 1.5 s max-wait sets the commit rate (0.64/s ≈ one per 1.5 s), so rows per -typing-second are ≈ 2 × 0.65 = 1.3 (counter) / 1.45 (gql), whatever the key rate. MB stream at 25 frames/s: -0.09 rows/edit. C2-stress 50k frames: 288 commits, 0.05 rows/frame. - -### 4.5 Step1/step2, catch-up, bootstrap, currentness - -- **Step1.** The server sends step1 with the head SV at accept. A client step1 is authority-checked first (G6, - round 3: a revoked actor gets 4403, not the body), then gets `step2 = diffUpdateV1(merged, clientSV)` from stored - bytes (ywasm stateless op, patch 0003, - `crdt/ywasmByteOps.ts`). No document is created. - - Merged bytes are cached. They are rebuilt with `durableMergedBytes` (latest checkpoint plus journal rows after the - checkpoint, up to the head, merged). The rebuild throws if the journal crosses an epoch without a checkpoint. - - Above `YAOS_RELAY_MAX_MERGE_INPUT_BYTES` (9 MiB) the server does not merge. It sends the stored parts - as one step2 per part (`replyStep2Unmerged`), and clients merge idempotently. -- **Catch-up** (`POST /catch-up`) and **GET body** use `relay.bodyHttpState`: merged bytes plus the catalog hash. - - If the catalog hash is NULL and the body is ≤ `lazyHashMaxBytes` (3 MiB), the server materialises it - once: `reconstructDocument` (a ywasm Doc) → `readText("body")` → canonical Markdown → sha256 → - `backfillCatalogHash` (UPDATE only while the hash is still NULL; lean mode updates the journal row's inline - hash). Round 3 (G4) adds a 256-body in-memory cache keyed by (epoch, head sequence), so repeated reads of an - unchanged unenveloped body do not rebuild the Doc (`lazyHashCacheHits`). - - Above that size the hash state is `unknown`. - - An over-budget body returns 413 `relay_merge_budget_exceeded`. -- **Bootstrap.** With the flag on, `bootstrap.ts` serves `durableMergedBytes(bodyId, boundarySequence)` as a byte - merge (round 2). Round 3 (G20) checks the persisted budget marker and returns 413 **before** reserving the batch, so - one over-budget note no longer fails a reserved batch midway. The protocol doc was aligned in round 3 (G17). -- **Currentness.** `BODY_CURRENTNESS_QUERY` and `HEAD /body/:id` read the head row and the catalog, with no merge and - no document. The hash may be `unknown`. - -### 4.6 Checkpoints - -- **Trigger.** Default mode: an append that pushes the tail to ≥ 50 rows or ≥ 1 MiB arms the alarm. Lean mode: every - append arms the alarm, delayed by `YAOS_RELAY_LEAN_CATALOG_DELAY_MS` (2 s), so a burst coalesces into one pass. -- **`alarm()` → `runCheckpointPass`** (round 3, G1), in this order: - 1. Lean only: `coalesceLeanCatalog` writes one catalog event per dirty body (3 rows per dirty body per pass). - 2. Evict stale residents. - 3. For each relay body over the thresholds that is neither a dirty base resident nor marked over budget, call - `relay.checkpointBody`. - 4. Set `floor = min(currentSequence − 1000, pinBoundary − 1 for every active pin)`. When that is above the current - floor, call `advanceFeedFloor(floor)`. This prunes journal rows at or below the floor that a complete checkpoint - covers. The floor does not advance if lean coalescing failed. - 5. Prune candidate receipts. - - The base `maintain()` loop now runs only for non-relay bodies. Measured in the unit suite: 5,000 relay appends leave - 1,001 journal rows (floor 4002, 41 floor advances). -- **`checkpointBody`.** - - If the tail is more than 200 rows, it merges only the first 200 (`tailPrefixSequence`) and writes a **partial** - checkpoint (`writeRelayCheckpointThrough`, `requireExactHead = false`). The partial prefix is itself checked - against the merge budget (G20). - - Otherwise it writes an exact-head checkpoint. - - **`checkpointsFromCache`** (round 4): when the merged cache entry is at the head, a full checkpoint uses those - in-memory bytes and skips the SQLite read and the Wasm merge. Local pass time for a 30 KiB body fell from 3.3 ms to - 1.8 ms (measured, local). - - Merges happen only through the stateless byte ops. Past the 9 MiB merge budget the body is skipped and marked in - `relay_body_budget` (persisted, G20) until a client reset. -- **`persistCheckpoint`** runs as one transaction: - 1. write the chunks and the manifest - 2. `assertActivePinRetainedCheckpointCapacity` - 3. delete journal rows ≤ `min(through, journalFloor())`. Lean mode calls `syncLeanClock` first. - 4. `pruneUnpinnedDocumentHistory` -- The alarm re-arms immediately while a checkpoint is partial or still needed. -- The base `writeLiveCheckpoint` path routes relay bodies to the same function. -- **Alarm cost and mb 0 tails** (round 4, deployed small n): a pass is about 60 ms CPU, with 0.5–4 s wall time - deployed. At mb 0 each frame is its own transaction plus an output-gate flush, and frames queue behind the alarm. - That is the source of the 0.4–2 s p90 at mb 0 against about 68 ms at mb 10 (MB smoke, v0930). Full n (measured, - MB sweep, section 8.1): stream p90 / p99 lean mb 0 92.9 / 416 vs lean mb 10 80.6 / 316 ms. -- **Alarm CPU grows with body history** (measured, full run): relay alarm p99 59 ms (C1), 108 ms (C2-quick) and - **1,041 ms** (C2-stress, 50k edits); strict up to 817 ms. Under the 30 s DO limit, but the output gate waits behind - it. Open (R9). - -### 4.7 Receipts - -- A receipt is the origin `BODY_COMMITTED` for an enveloped frame. It is sent only after the transaction commits. -- It carries `vaultSequence` and `durableGeneration`, plus `candidateId`/`candidateDigest` when the frame had them. - The same row lands in `vault_candidate_receipts`, so an HTTP candidate replay or a socket resend of that candidate - resolves to the same receipt. Since round 3 (G14), `GET /operations/:candidateId/outcome` also resolves. It uses the - attribution `operation_id` by default, or the inline journal columns in lean mode. -- **HTTP candidate under v3 (14e5250, inferred, not deployed).** With group commit on, a non-creation HTTP candidate - commits through `relay.commitHttpCandidate`: 1 tail record + 1 head + 1 `relay_device_receipts` ring entry = - **3 rows** (base path 14: clock 1, journal 2, attribution 2, head 2, catalog_events 3, operation_outcomes 2, - candidate_receipts 2). A no-op save writes only the ring entry (1 row). The catalog event is coalesced by the - checkpoint alarm (≤ 5 rows per body per 30 s window). Replay, digest reuse (409) and - `GET /operations/:id/outcome` resolve from the ring (256 per device; older replays rely on CRDT idempotence). - The receipt JSON is unchanged, so the client needs no change. Creations still use the base path. -- D7: `src/sync/vaultSync.ts` is not modified in this spike. The harness implements socket-ack receipts - (`scripts/relay2/lib/socketReceipts.ts`). -- Integration sketch: - - `VaultSync.create({ webSocket })` receives a WebSocket factory, and the relay wrapper's `send()` injects the - envelope before each binary update. `sync/fencedWebSocket.ts` is the natural home. - - The candidate observer maps each IDB candidate to `candidateId = candidate id`, `candidateDigest = candidate - digest` and `clientFrameId`. - - The candidate is cleared on the relay `BODY_COMMITTED` whose `candidateId` and `candidateDigest` match. The HTTP - POST remains the fallback after a socket loss. -- **v3 receipts** (group commit): a receipt arrives when the group commits, **0.3–1.5 s after the edit** (user-approved; - 300 ms idle, 1.5 s max). Measured (relay3): C4-v3 ack p50 / p99 351 / 625 ms at 1 edit per 1.5 s; MB type5 ack - p50 / p99 907 / 1,642 ms at 5 keys/s (the 1.5 s max-wait bounds it, plus RTT); L2-v3 origin ack 350 / 481 ms - (n = 290). Acks are cumulative per socket (invariant 3). The native `VaultSync` `RelayReceiptChannel` - (`relayReceipts`, landed with v3, `1a34693`) resends after 5 s without a receipt; the harness client does the same - (`--resend-ms`). - -### 4.8 Budgets - -| Budget | Default | Enforced at | -|---|---|---| -| Frame size | 1,750,000 B (+64 outer); text 64 KiB | raw gate, first step of `VaultSocketService.message` (close 1009) | -| Per-socket rate / burst | 256 KiB/s on raw message bytes, envelopes included; burst `max(1 MiB, 1,750,064 B + 64 KiB)` | raw gate (`admitKnown`; close 1013, later messages dropped O(1)) | -| Group-commit min interval | 1,000 ms (`YAOS_RELAY_GC_MIN_INTERVAL_MS`, 0 = off) | idle timer (forced flushes exempt) | -| Micro-batch window | 0 ms (clamped 0..250; full-run relay config 10) | `handleSyncFrame` / batch timer | -| Relay + semantic sockets per vault | 5,000 | `acceptRelayBody` (429) | -| Merged-bytes cache | 16 MiB LRU | `remember` | -| Exact-merge threshold | 256 KiB | `commitFrames` | -| Checkpoint trigger / rows per pass | 50 rows or 1 MiB / 200 (lean: every append, delayed 2 s) | alarm, `checkpointBody` | -| Feed retention | 1,000 sequences below the current one (pins respected) | `runCheckpointPass` | -| Lazy-hash cache | 256 bodies | `bodyHttpState` | -| Merge input budget | 9 MiB | `durableMergedBytes(…, maxMergeInputBytes)` | -| Lazy-hash materialisation | ≤ 3 MiB | `bodyHttpState` | -| Lease TTL | 120 s default, 1–600 s | `acquireLease` | -| Reset cooldown | 24 h (0 in the full-run configs) | `acquireLease`, `semanticReset` | -| Reset snapshot | JSON ≤ 8 MiB (`MAX_CATCH_UP_BYTES`); octet-stream ≤ `RELAY_MAX_RESET_SNAPSHOT_BYTES` | `readResetInput` | - -### 4.9 Semantic reset lease - -Only clients can do history GC, because it needs a fresh `Y.Doc` built from the text. - -**Acquire.** `POST …/compaction-lease` with `{expectedEpoch, ttlMs}`. `RelayBodyStore.acquireLease` runs one -transaction: -1. the head exists -2. its epoch equals `expectedEpoch` -3. the cooldown has elapsed since `last_compacted_at` -4. the actor is re-validated inside the transaction (G19, round 3) -5. no other device holds an unexpired lease -6. upsert the lease - -Responses: -- 200: `leaseId`, `expiresAt`, `epoch`, `headSequence`, `generation`, `stateVector`, `policy` -- 409: `held` or `epoch_mismatch` -- 404: `not_found` -- 429: cooldown, with retry-after (measured on coresmoke-1: `retry-after: 1` with a 1 s test cooldown) - -`{release: leaseId}` deletes the lease row for the same device. - -**Build (client).** The client builds a lineage-fresh doc from the current text (K2). It optionally pulls -server-sourced state after taking the lease. - -**Install.** `POST …/semantic-reset`, as JSON with a base64 snapshot or (round 2) as octet-stream with the -`x-yaos-lease-id`, `-expected-epoch`, `-covered-sequence`, `-content-hash` and `-content-bytes` headers. Checks: -1. `snapshotStructurallyValid`: 2 B ≤ size ≤ limit, the SV parses, and the SV is non-empty unless `contentBytes = 0`. - This replaced the committed `snapshotCoversHead`, which rejected every lineage-fresh reset. -2. The lease `lease_id` and `device_id` match; it is unexpired; its epoch equals the head epoch, which equals - `expectedEpoch`. -3. `coveredSequence === head.latest_sequence` exactly. -4. Cooldown, and the actor is re-validated inside the reset (G19). Pending micro-batches for the body are flushed - before the CAS (G3). -5. `semanticResetFromEncodedState` runs as one transaction: - - exact-head assert - - refuses while a creation is pending or the lifecycle is unpublished - - `vault_clock + 1` - - journal row `'semantic-reset'` (0 B) - - checkpoint chunks and manifest at the new sequence and the new epoch - - pin capacity check - - head CAS on (generation, epoch, sequence) → epoch + 1 - - catalog event with `COALESCE(client hash, previous)` -6. Delete the lease. -7. `discardResident`, then `fenceSemanticEpoch`: every old-epoch socket, relay included, is closed with 4409. -8. Invalidate the merged cache. - -Errors: 409 for `lease_invalid`, `lease_expired`, `epoch_mismatch`, `head_advanced` and `cooldown`; 400 for -`invalid_snapshot`. - -**Losers.** -- A client holding old-epoch edits gets 4409, fetches the new epoch and rebases its text intent. The spike rebase uses - `mergeThreeWayText`, with a no-intent fast path. -- Server-side semantic compaction (`semanticCompactionRuntime.recordCommit` / `documentLoaded`) is skipped for relay - bodies (`server.ts` `afterDurableCommit`, `afterDocumentLoaded`). - -### 4.10 Revocation, epochs, runtime identity - -- **Revocation.** `closeDevice` / `closePrincipal` close every socket of the device or principal with 4403, relay - sockets included. Authority writers bump `authorityVersion`, so the next frame on any surviving socket fails - `validateActorCached`. Pings and currentness queries re-validate too. -- **Delete.** `closeBody` closes relay sockets with 1008. The append transaction also refuses non-`active` catalog - heads. -- **Epochs.** The ticket epoch is checked at accept (`SemanticEpochMismatchError`). The head epoch is checked on step1, - at commit, and inside the transaction. A reset fences every old-epoch socket. -- **Runtime identity.** - - `runtimeEpoch` is random per `VaultRuntime` construction. - - Relay attachments keep their admission epoch. `message()` does not apply the runtime-epoch mismatch close to - relay attachments with the same vault ID and generation (`vaultSocketService.ts` ~L749). - - Receipts carry both the admission `runtimeEpoch` and `commitRuntimeEpoch`. - - Durability does not depend on the runtime. The receipt is issued after `transactionSync`, and a runtime restart - only loses in-memory state: pending envelopes, the rate buckets, micro-batch queues, and the cache. - -## 5. What moves where - -| Responsibility | Today (base) | Relay | Rationale | -|---|---|---|---| -| Body frame admission | Resident ywasm doc plus validation mirror doc per body (`vaultDocumentCache.ts`); `bodyUpdateAdmissionError` (frontmatter root shape, 5 MB Markdown cap) | Byte cap, epoch, authority, rate. No document. | The Wasm envelope was what capped sockets at 32 and cost 19 ms on heavy notes. | -| **Body content validation** | Server, before persistence | **Receiving clients, before their disk write. Owned elsewhere; out of scope for this spike.** | Disk is the truth and the client owns it. The server cannot validate once E2EE exists. | -| Persistence timing | 250 ms throttled flush (`scheduleFlush` → `flushDocument` → `commitUpdate`) | Synchronous append per frame, or per micro-batch (0..250 ms; recommended 5–10) | Removes ≈ 250 ms of latency. Costs more appends per keystroke (section 9); lean rows offset most of it. | -| Ordering | `vault_clock` per flush | `vault_clock` per append (lean: `MAX(clock, journal max) + 1`, no clock write) | Unchanged meaning. | -| Attribution | Per flush | Per frame (N rows per append; lean: frame 0 inline on the journal row) | More precise. Lean prunes frame-0 attribution at the feed floor. | -| Broadcast | After flush commit, from the resident doc | After append commit, same bytes | Invariant 1 is preserved. No re-encoding. | -| **Candidates (second write path)** | Every edit is also an IDB candidate POSTed over HTTP, cleared by `DurableReceipt` | The socket ack is the receipt (same row in `vault_candidate_receipts`). The HTTP POST becomes a fallback only after socket loss. The creation-candidate fence stays on HTTP. | Removes one Worker request plus a DO hop per burst, and a second journal write of the same bytes. | -| Step1/step2 | ywasm doc `encodeStateAsUpdate(sv)` | `diffUpdateV1` on stored bytes | No document. K3: 0.26–14 ms for 1–10 MB. | -| Catalog hash / size | Computed by the server from the doc on every flush | Client-claimed in the envelope and accepted only when the claim SV equals the merged SV. Otherwise NULL, then lazily materialised on read (≤ 3 MiB, 256-body cache). Lean: the hash sits inline on the journal row, and catalog events are coalesced by the alarm. | The server does not read plaintext on the hot path. | -| Routine checkpoint | `maintain()` via a ywasm doc | Byte merge through the stateless op (alarm, ≤ 200 rows per pass) | Plaintext structure is not interpreted. Bounded CPU. | -| **Semantic compaction (history GC)** | Server (`semanticCompaction*.ts`, fresh doc + epoch bump) | **A client under a lease** builds and uploads; the server CASes the epoch | Only clients can compact past the 96 MiB Wasm cap (the 5m fixture traps on the server). Required for E2EE. | -| Catch-up / GET body | Reconstruct doc or resident doc | Merged stored bytes (+ lazy hash) | No document except the lazy-hash fallback. | -| Bootstrap | `reconstructDocument` | Byte merge through the boundary sequence; 413 before reserving when over budget | Same. | -| Currentness / HEAD | Head + catalog | Head + catalog; the hash may be `unknown` | Unchanged, plus an explicit hash state. | -| Hibernation survival | Base closes with 1008 after eviction (runtime-epoch fence) | Relay socket survives | Idle product shape. | -| Revocation, membership, namespace, lifecycle, root doc | Authoritative server | **Unchanged**, authoritative server | Authority lives in the namespace. | -| Canvas semantic documents | Server-validated semantic docs | **Unchanged** | Out of scope. | -| Rate limiting | Cache pressure and backpressure | Per-socket token bucket on raw bytes, checked first in the DO message handler, then 1013 `relay rate limit` (distinct from platform `Service overloaded`). After the upgrade, WebSocket messages go straight to the DO and never pass the Worker, so there is no earlier enforcement point. | Explicit budget. | -| Socket cap | 32 (memory envelope) | 5,000 (config) | Bytes, not docs. | - -## 6. Invariants - -| # | Invariant | How it is enforced | Code | Gaps / status | -|---|---|---|---|---| -| 1 | **v2:** no edit is broadcast to peers before its append transaction commits (durable before broadcast). **v3 (`YAOS_RELAY_GROUP_COMMIT`): durable before receipt.** A frame is broadcast as soon as it passes admission; its `BODY_COMMITTED` receipt is sent only after the group commit that contains it returns. HTTP reads (GET/HEAD, bootstrap, feed) only ever see durable state. Precedent: base broadcast before its debounced flush until `e1ae3a4`. | v2: fan-out and `BODY_COMMITTED` only after `appendRelayBodyUpdate` returns. v3: `groupEnqueue` broadcasts, `flushGroup` commits in one `transactionSync` and then acks. A frame lost with the buffer (crash, eviction) was never acked, so the origin resends it (client resend after 5 s without a receipt, and on reconnect), and wake re-sync (`ensureWakeResync`) pulls it back from any peer that applied it. | `relayBodies.ts` `commitFrames` (v2), `groupEnqueue`/`flushGroup`/`ensureWakeResync` (v3) | v2: none found. v3 (measured, relay3 run r3-1002b): CRASH 10/10 rounds (5 origin-online, 5 origin-offline with recovery from the peer), 5–6 buffered frames dropped per round, 0 frames lost, fresh device C converges. Open: a revoked device's buffered frames reach peers but are dropped at flush (row 4, R11). | -| 2 | Every append has exactly one vault sequence, and the feed returns it after the cursor. | Default: `vault_clock` is bumped in the same transaction as the journal insert. Lean: the sequence is `MAX(clock, journal max) + 1` inside the transaction, and the clock is synced before any journal delete. The sequence is the journal PK. | `relayBodyStore.ts` `appendRelayBodyUpdate`; `syncLeanClock` | A reset uses one sequence too. Feed retention is 1,000 sequences (G1 fixed, round 3). In lean mode the catalog delta feed lags by ≤ 2 s. | -| 3 | A returned receipt means the bytes are durable. **v3 adds: acks are cumulative per socket.** An ack for frame N confirms every earlier frame of that socket. | The origin ack is sent after `transactionSync`. A dedupe re-ack reads a committed receipt row. A no-op ack means the bytes are already contained in durable state. v3: one body and epoch per socket, one buffer per (body, epoch), synchronous byte-cap flushes and one transaction per group keep commits in arrival order. A refused frame (rate, too large, authority, epoch, inactive body, commit failure, frame error) **fences** its socket: later frames are dropped unacked (`failedSocketDrops`), earlier buffered frames still commit. After a wake, an earlier-runtime socket's acks are held until its re-sync step2 is durable (`wakeHeldAcks`). | `commitFrames`, `ackOrigin`, `ackNoop`; v3 `markFailed`, `afterFailure`, `screenBatch` | The incremental-SV no-op for large bodies only matches identical bytes, which is safe. v3 fence (measured, r3-1002b/c): too-large 5/5 (1009; nothing after the refused frame acked or durable); rate 6/6 rounds that drew our 1013 (0 acked and 0 durable after the refused frame; `failedSocketDrops` 10–76). The ack prefix held in every v3 run (MB, C4, FENCE, CRASH). | -| 3a (round 4) | No frame is dropped silently. Every non-empty update frame ends appended, acked as a justified no-op or dedupe, or with the socket closed. | Outcome counters satisfy the frame-accounting identity (4.3). Pre-append throws close 1011. | `relayBodies.ts` `failFrames`; `vaultSocketService.ts` `relayMessage` | Covered by a random-interleaving test at mb 0 and mb > 0, and checked on full-n diagnostics deltas. | -| 4 | A revoked device's frames after the fence sequence are never appended. | Sockets are closed with 4403 on revoke. `validateActorCached` runs on every frame, on step1 (G6), and again at micro-batch flush (G2). `authorityVersion` is bumped synchronously by the writer. The 5 s TTL applies only to cross-isolate changes. | `closeDevice`; `validateActorCached`; `commitFrames` | G2, G6 and G19 fixed in round 3. Full n: B4-relay 4403 after 164 ms, 0 appends after (measured). v3 (measured, B4-v3): 4403 after 183 ms, 0 appends after the revocation started, so row 4 holds; but 10 frames sent *before* the revocation were broadcast, then dropped at flush (G2), so the live peer holds text the server does not, and the convergence check fails (fresh C and HTTP GET differ from the live clients). See R11. | -| 5 | Old-epoch frames are never appended after an epoch bump. | Ticket epoch at accept. Head epoch before commit, per (body, epoch) batch. `expectedEpoch` inside the transaction. Reset flushes pending batches and then `fenceSemanticEpoch` closes old sockets. | `commitFrames`; `appendRelayBodyUpdate`; `handleSemanticReset` | G3 fixed (round 3). | -| 6 | Flag off means baseline behaviour. | Every relay branch is gated on `this.relay !== null` / `relayBodiesEnabled`. | `server.ts`, `vaultSocketService.ts`, `bootstrap.ts` | Measured (`flag-tests.md`). Round 4 flag-off: 3,635 assertions pass, 0 new failures. Round 1: 199/199 steps. | -| 7 (D6) | A catalog content hash is recorded only if the claimant's SV equals the merged SV after the append, and that SV is exact. | `stateVectorsEqual(envelope.stateVector, nextStateVector) && stateVectorExact`. Otherwise NULL. | `commitFrames` | G8 fixed (round 3): bodies over 256 KiB never accept a claim. The hash is still unverified client data (G10). Smoke CW: 21/21 relay, 21/21 base (v0930). Full n (measured): CW-relay 119/119 (104 materialised, 15 known), CW-base 121/121. The X3-relay "VIOLATED" is a `checks.ts` artifact (empty-string header treated as a claim). | -| 8 (added) | Idempotent retries: one append per (device, candidateId), always with the same receipt. | Receipt lookup after the authority check (G7). In-batch duplicate detection (G11). `ON CONFLICT DO NOTHING` inside the transaction. | `handleSyncFrame`; `appendRelayBodyUpdate` | G7 and G11 fixed. In lean mode a candidate resend older than about 1,000 sequences and its receipt is re-appended, which is a CRDT no-op on the exact path. Full n (measured): B6-relay appendsDelta 1, dedupeHits 3, same receipt. | -| 9 (added) | The server never calls into Wasm with more than the merge budget of input. | `durableMergedBytes(…, maxMergeInputBytes)` throws `RelayMergeBudgetError` before merging. The lazy hash is capped at 3 MiB, and partial checkpoint prefixes are bounded (G20). A ywasm OOM trap poisons the instance, so the refusal happens before the call. | `durableMergedBytes`; `bodyHttpState`; `checkpointBody` | G5 fixed: `syncDocumentCache` no longer applies updates. | -| 10 (added) | A checkpoint never loses a journal row. Journal rows are deleted only in the same transaction that writes a checkpoint covering them, and never above the feed floor. | `persistCheckpoint` deletes `≤ min(through, journalFloor())` inside the checkpoint transaction. The floor advances only in `runCheckpointPass`, and stays below every pin. | `persistCheckpoint`; `runCheckpointPass` | G1 fixed. Measured locally: 5,000 appends leave 1,001 rows. Full n (measured): K1-compact-relay → 0 log rows; C2-stress relay/strict converge at 50k. Per-body journal row counts after the stress trace: not reported in the full-run tables. | -| 11 (added) | At most one semantic reset per epoch. | Lease upsert is one transaction. Reset requires the lease, an epoch CAS, the exact head sequence, and a head CAS inside `semanticResetFromEncodedState`. | `acquireLease`, `semanticReset` | Measured: B5 6/6 (e1abb36); full n B5-relay 20/20, exactly one install per race. | - -## 7. Failure modes - -| Case | What happens | Residual risk | -|---|---|---| -| **A client bug writes bad bytes** (malformed or structurally wrong Yjs, bad frontmatter root) | The server does not parse structure on the hot path, so it appends any update of 1.75 MB or less. Malformed bytes can make the next `mergeUpdates` or `diffUpdate` throw. That means step1, checkpoint or HTTP read failures for that body only. The error is caught per body, the alarm re-arms, and the body stays checkpoint plus tail. Semantically wrong but valid updates, such as a bad frontmatter shape, propagate to peers. Receiving-client validation (owned elsewhere) must refuse to write them to disk and surface a conflict. Recovery: a client semantic reset from good text (lease) replaces the lineage, and the history is kept per retention and pins. | Until receiving-side validation ships, relay mode loses today's server-side frontmatter/root-shape gate. The failure is per body and does not spread to the vault. There is no poison-pill quarantine yet (open question Q3). | -| **The lease holder dies** | The lease expires after its TTL (120 s default). No state changed, because reset is a single transaction at install. Another device can acquire the lease after expiry. The holder can release early. | Fixed in round 3 (G19): revocation and fencing delete the holder's lease rows, and the actor is re-validated inside lease and reset. | -| **Two resets race** | The lease upsert serialises them: the second device gets 409 `held`. If a lease expires mid-upload (B5: 5 MB upload at 115.7 s), `lease_expired` rejects the install. At install, the epoch CAS plus `coveredSequence == latest_sequence` plus the head CAS let exactly one win. The losers' sockets get 4409 and they rebase. | Measured B5: 6/6 (component), and 20/20 at full n (B5-relay), alternating winners, one install per race. Starvation risk: R2 (not measured). | -| **DO crash mid-append** | `transactionSync` is atomic: either all rows (clock, journal, attribution, head, catalog, receipt) or none. No ack or broadcast happens before commit. On crash the socket drops, the client reconnects and resends (step2), and the growth cap or receipt deduplicates. | An in-memory micro-batch (≤ 250 ms, recommended 10 ms) is lost, but it was never acked. The client resends unacked frames. **v3:** the group buffer (≤ 1.5 s) is lost, but peers may already have applied it (durable before receipt). The origin resends after 5 s or on reconnect, and wake re-sync pulls the frames back from any peer. Measured: CRASH-v3 10/10 rounds, 0 frames lost, recovery p50 ≈ 0.8 s; origin offline, recovery from the peer, 5/5. Residual: a frame that reached no peer and whose origin never returns is lost, as in v2 (it was never acked). | -| **DO crash mid-checkpoint** | `persistCheckpoint` is one transaction: chunks, manifest, journal deletes and prune together. After a crash either the old checkpoint plus the full tail remains, or the new checkpoint with the tail pruned. The alarm re-runs. A partial checkpoint (≤ 200 rows) is itself a complete, consistent checkpoint through its sequence. | none found | -| **Log growth while all devices are offline** | Without appends no alarm is armed, so nothing grows. With one writer and no readers, each append arms the checkpoint at 50 rows or 1 MiB, so the merged snapshot stays bounded by live size plus tombstones. Journal rows below the checkpoint are deleted only up to the feed floor. | G1 fixed in round 3: `runCheckpointPass` advances the floor to `current − 1000`, below every pin. Over-budget bodies (> 9 MiB merged input) are never checkpointed until a client reset. Their marker is persisted (G20). Full-n check: K1 and `C2-stress-relay` table counts. | -| **Mobile-only vault** | The routine checkpoint is server-side, so no client is needed. Reset is feasible on phones: K2 inferred 48–586 ms build, duty-cycle bound 1.7 s p50 for 5 MB. Upload speed is the constraint. Before round 2, B5's JSON 5 MB upload took 5–116 s. After round 2, a binary 5.24 MB reset takes 1.7–3.3 s from a desktop (core3, n = 4), with no growth across iterations. The lease TTL (≤ 600 s) and the octet-stream upload (which avoids the 33% base64 overhead) matter here. | The exact-head CAS combined with a slow mobile upload on a hot note is R2. | -| **Hibernation** | Relay attachments keep the admission `runtimeEpoch` and are exempt from the mismatch close. The merged cache rebuilds from SQL on the first frame. Measured full-n B1: the relay socket survives 150 s idle plus 2 restarts (afterIdle 118 ms, afterRestart2 71.9 ms); base loses all 4 sockets with 1008 "socket authority mismatch". | Lost on wake: pending envelopes (G13, memory-only by design and documented as a deviation; the next echo is unenveloped), rate buckets (they refill), and the micro-batch queue (unacked, so it is resent). | -| **Revoked device with an offline queue** | On reconnect the ticket issue fails, because the control plane rejects the revoked credential. If a socket survived, the first frame fails `validateActorCached` with 4403. Queued edits never append. Attribution rows record device and credential revision for anything appended before the revocation. | The revoked user's local edits stay local by design. G7 fixed in round 3: authority now runs before dedupe, so there is no receipt re-ack for a revoked device. | -| **Transport loss mid-stream / DO stall** (round 4 smoke data-loss finding) | Root cause in smoke mb0, worker v0930: the transport/platform plus harness behaviour. Server merge logic was not involved.
• l2: the DO stalled (alarm `internalError` after 27 s wall, 2 hibernation `internalError`s). Both sockets died about 49 s into a 60 s drain. All 61 edits were durable, but the raw harness client never reconnected, so B missed #37–60.
• burst: the sender's socket ended `clientDisconnected` about 38 s in. The raw client silently skipped sends while the socket was not OPEN, so 27 tail frames never reached the server.
Repros were clean, and the keystroke fuzz (n = 61 / 241 / 2,000) was clean. A real client must reconnect, re-sync with step1, and resend unacked frames. The harness now does all three (35b2882). | Server side: one latent silent drop was found and fixed (4.3, `frameErrors`). Client side: `VaultSync` integration must keep resending until it receives a receipt (phase 2). | -| **Platform overload** (`1013 Service overloaded`) | Cloudflare sheds the connection when the DO queue or CPU saturates. Seen once in K1 at mb 0 under a flood. The server never sees it, so no counter moves. | Mitigated by micro-batching (fewer transactions and output-gate flushes). Clients treat it as a reconnect and resend. Full n: recurred at X2-strict 400/s on 1 body (1 reconnect, 1,520 frames resent); B7 closes were our own `relay rate limit`. | -| **Malicious member** (valid credential, hostile client) | Rate limit (1013) and frame cap bound flooding. A malicious member can: append arbitrary valid Yjs, which propagates, just as with base once server validation is gone; claim false content hashes that equal the merged SV (the server cannot verify, G10), poisoning catalog hashes and currentness until the next honest claim; take the lease and install a reset with arbitrary text (the history is kept per retention; the reset is attributable); and hold leases to block resets. | Same trust model as today for members: members can write content. Hash poisoning (G10) is new: today the server computes hashes. Mitigation options: receiving clients verify hashes and report mismatches; the server re-hashes on the lazy path whenever a hash is disputed. | - -## 8. Measurements - -Dates, Workers, deploy version IDs and spike SHAs come from each run's JSON metadata (full run f1001, 2026-10-01; -lane B 13:37–13:58 UTC, reused phases from 15:07 UTC). Full-n raw output is -`experiments/logs/relay2/runall-f1001/raw/.json`; the index is in section 14. "Base" means the same branch -and the same vars with the flag off. "relay" means lean + mb 10, and "strict" means lean off + mb 0. Every full-n cell -is **measured** unless labelled otherwise; latency n excludes 10 discarded warm-up samples. Concurrency labels are -defined at the top of this document. - -Small-n values marked "smoke" come from v0930 (`8e194a4`, default rows) or w0930 (`276c16c`, primary configs). They -are directional only. - -### 8.1 Latency - -| ID | Metric | Base p50 / p90 / p99 | Relay p50 / p90 / p99 | Strict p50 / p90 / p99 | n | Notes | -|---|---|---|---|---|---|---| -| L1 | ping RTT | 50.0 / 53.9 / 247 | 50.2 / 60.2 / 115 | — | 90 | [serial]. Floor; v1 45.4 / 50.1 ms (A5) | -| L2 | propagation (+ origin ack) | 308 / 334 / 608 (ack p50 309) | **66.0 / 100 / 314** (ack p50 65.8) | 56.2 / 115 / 333 (ack p50 56.2) | 290 | [serial]. **4.67× p50**, p99 0.52× base: target met. Strict 5.48× p50. Seq delta base 300, relay 373 (lean catalog coalescing), strict 300. v1: 300 / 572 → 51.8 / 279 (n = 200) | -| L3 | propagation on 64k chars, before / trace replay per frame / after | 304 / 322 / 614; 261 / 366 / 597; 340 / 349 / 356 | 66.9 / 158 / 400; 71.5 / 153 / 504; 71.2 / 102 / 618 | — | 40 / 5,000 / 40 | [serial]. Relay p99 after (618) ≈ base before (614); p50 4.8× better after. v1: 302 → 318.7 vs 52 → 52.5 ms (A5) | -| L4 | per-frame at 25 edits/s (quick trace) | 213 / 313 / 543 (max 785) | 73.7 / 532 / **2,305** (max 3,435) | 60.0 / 273 / 866 (max 2,183) | 4,990 | Base/relay **[conc]** with L5–L7; strict [serial]. Relay tails (0.3–3.4 s) recur throughout the run, including after L6/L7 ended; p50 per 250-frame window ≈ 70 ms. Same config in lane B (MB-lean-mb10-stream, n = 2,750, [B∥10]): 63.3 / 80.6 / 316. **Relay p99 tail is open** (section 12, R9). v1: 226 / 688 → 51.4 / 757 | -| L5 | edit → cleared receipt (HTTP candidate vs socket ack), 1-edit burst p50 / p90 | prod 480 / 553; nodebounce 237 / 326 | relay **77.0 / 357**; relay250 322 / 546 | — | 90 per mode | [conc]. 6.2× vs prod. DO units/burst: base prod 1.05 (1 HTTP req), relay 0.10 (0 HTTP). relay250 shows the debounce explains most of the base gap | -| L5 | 8-edit burst, first / last edit p50 (p90) | prod 1,356 / 475 (1,393 / 508); nodebounce 1,165 / 280 (1,309 / 416) | relay 959 / 67.7 (1,006 / 116); relay250 1,217 / 333 (1,305 / 425) | — | 90 per mode | [conc]. DO units/burst: base prod 1.40, nodebounce 5.40 (5 HTTP); relay 0.80, relay250 0.45 (0 HTTP) | -| L6 | open → synced, cold / warm | cold 365 / 396 / 496; warm 363 / 402 / 710 | cold 362 / 393 / 451; warm 369 / 397 / 776 | — | 100 cold / 90 warm | [conc]. ≈ equal | -| L7 | HEAD after ack; edit → HEAD shows edit | 218 / 242 / 275; **758 / 834 / 1,343** | 211 / 228 / 539; **505 / 580 / 815**; hash correct in all | — | 90 | [conc]. GET after ack: base 217 / 239 / 266, relay 214 / 233 / 307. 1.5× faster edit → visible | - -**Micro-batch sweep (MB).** Propagation p50 / p90 in ms, for l2, burst and stream patterns. Smoke, v0930, default rows, -small n, measured: - -| Config | l2 | burst | stream | -|---|---|---|---| -| base | 310 / 403 | 301 / 413 | 200 / 307 | -| mb0 | 52 / 281 | 54 / 660 | 53 / 1996 | -| mb10 | 65 / 74 | 62 / 68 | 62 / 66 | -| mb50 | 105 / 113 | 102 / 130 | 101 / 145 | -| mb100 | 153 / 156 | 151 / 155 | 115 / 152 | -| mb250 | 303 / 2343 | 187 / 309 | 189 / 305 | - -Full n (measured, f1001, [B∥10]; trimmed with user approval to base, strict and lean × mb {0, 10, 50}; mb100, mb250 -and nocand were **not run at full n**, smoke values above only). n = 881 frames per burst run and 2,750 per stream run. -Rows/edit is gql, idle-subtracted; CPU is periodic µs per edit (idle-subtracted phase µs in brackets). Every relay run -passed the frame-accounting identity. - -| Config | Burst p50 / p90 / p99 | Stream p50 / p90 / p99 | Rows/edit burst / stream | CPU µs/edit burst / stream | Source | -|---|---|---|---|---|---| -| base | 203 / 323 / 546 | 246 / 467 / 1,046 | 5.10 / 3.10 | 2,528 (2,335) / 6,550 (6,599) | `MB-base-{burst,stream}` | -| strict (full rows, mb 0) | 56.5 / 78 / 701 | 60.1 / 132 / 506 | 11.23 / 11.84 | 2,053 (2,306) / 3,866 (4,076) | `MB-strict-*` | -| lean mb 0 | 66 / 116 / 539 | 54.1 / 92.9 / 416 | 5.55 / 5.95 | 1,728 (1,212) / 2,314 (2,054) | `MB-lean-mb0-*` | -| **lean mb 10 (relay config)** | 63.4 / 132 / 517 | **63.3 / 80.6 / 316** | 5.54 / 5.92 | 1,979 (1,421) / 2,664 (2,422) | `MB-lean-mb10-*` | -| lean mb 50 | 106 / 310 / 1,711 | 114 / 145 / 463 | 5.54 / 4.88 | 2,495 (2,260) / 1,528 (1,367) | `MB-lean-mb50-*` | - -Reading: mb 10 does not reduce rows at typing rates (5.54 vs 5.55). Its value is p99 (316 vs 416 stream) and -output-gate smoothing. mb 50 doubles p50 and saves only ≈ 1 row/edit when streaming. - -### 8.2 Server cost - -| ID | Metric | Base | Relay | Strict | n / source | -|---|---|---|---|---|---| -| C1 | propagation small / heavy p50; WS CPU per invocation p50 / p90 / p99 (gql); checkpoint alarm CPU p50 / p90 / p99 | 303 / 385 ms; 1.36 / 2.27 / **64.3 ms** (5,101 inv.; HTTP CPU p99 244 ms); n/a | 73.9 / 90.8 ms; 0.22 / 2.92 / 4.19 ms (10,188 inv.); alarm 3.4 / 6.8 / 59.3 ms (n = 105) | 70.1 / 60.2 ms; 0.46 / 3.74 / 5.70 ms; alarm 52 / 81 / 98 ms (n = 96) | n = 40 each, [B∥10]. Phase CPU base 18.9 s, relay 12.3 s, strict 23.0 s. Heavy prop 4.2× faster, WS p99 15× lower. Tail files matched 0/40 (R6), not used. v1: heavy 19 ms vs ≈ 0 ms (A5) | -| C2 | quick trace (5,000 frames): WS CPU p50 / p90 / p99; µs/edit; rows/edit; propagation p50 / p99; alarm CPU | 2,077 / 16,830 / **30,697 µs** (p90 and p99 > 10 ms); 4,786 µs/edit; 3.10 rows/edit; 243 / 704 ms | 381 / — / 4,639 µs; 3,325 µs/edit; 6.20 rows/edit; 64.0 / **9,132 ms**; alarm 5.22 / 54.8 / 108 ms | 1,506 / — / 5,722 µs; 4,271 µs/edit; 12.19 rows/edit; 68.7 / 790 ms; alarm 22 / 79 / 102 ms | n = 5,000, [B∥10], max-rate lane B. Accounting PASS relay/strict. Relay propagation p99 9.1 s is open (R9). Smoke w0930 (300 frames) WS p99 was 11.2 / 3.2 / 2.2 ms | -| C2 | stress trace (50k edits, 5 clients, max-rate replay, latency not used): WS CPU p50 / p99; µs/edit; rows/edit; alarm CPU | 1,429 / 7,342 µs; 2,181 µs/edit; 2.02 rows/edit — **provisional**: 10,226 frames only (gql window incomplete; base semantic reset at frame 10,226), not comparable | 242 / 2,829 µs; 3,375 µs/edit; 5.84 rows/edit; **alarm p90 760 / p99 1,041 ms** | 439 / 1,855 µs; 7,271 µs/edit; 13.03 rows/edit; alarm p50 445 / p99 817 ms | n = 50,000 relay/strict, [B∥10]. Accounting PASS relay/strict. Alarm CPU grows with body history (R9) | -| C3 | Wasm linear memory / resident docs at 1, 8, 32, 100 bodies; 32 × 512 KiB | 32 resident; 1.51 MB at 32 small; **37.22 MB at 32 × 512 KiB**; the 100-small step is capped at 32 sockets (6 open failures); opens p50 / p90 570 / 712 ms | 0 resident; 1.18 MB at 100 small; **4.06 MB at 32 × 512 KiB**; 0 failures; opens 553 / 720 ms | — | steps 1/8/32/100, reused worker. 9.2× less. v1: base 37.5 MB vs relay 1.11 MB (A5) | -| C4 | rows written per edit / per reconnect; sequences per edit (50 edits, 20 reconnects) | 3.10 rows/edit (MB/C2 gql); 0 / reconnect; 1 seq/edit; ack 321 / 414 ms | **5.00** (relay counter; 5.54–5.92 gql in MB); 0 / reconnect; 1.5 seq/edit (lean catalog events); ack 69.6 / 103 ms | **11.00** (relay counter; 11.23–11.84 gql in MB); 0 / reconnect; 1 seq/edit; ack 55.0 / 68.6 ms | n = 50 + 20, [B∥10]. Whole-phase gql rows are dominated by the seed, so the relay counter is the per-edit source. **Component, measured:** 9 / 11 CF rows per plain / candidate append in default mode (coresmoke-1, n = 20 each), 3 / 5 in lean (corelean-1, n = 20 each). Smoke w0930 C2-quick rows per edit: base 3.03, relay 7.26, strict 13.16. Smoke C4 sequences per edit: 1 / 1.5 / 1, and per reconnect 0 / 0 / 0. | -| C5 | DO units per edit burst / per catch-up (20:1); rows per burst / catch-up | 0.51 / 1.92 units; 43.05 / 47.95 rows (phase-level idle-subtracted 0.72 / 2.28 units, 44.08 / 50 rows) | 1.88 / 3.43 units; 45.93 / 56.45 rows (phase-level 2.13 / 3.79, 46.95 / 58.50) | — | n = 40 bursts / 20 catch-ups, [B∥10]. Propagation 299 vs 65.2 ms; catch-up open 380 vs 367 ms. In absolute terms both are small (section 9: DO requests 2–3% of Free). Smoke v0930 (default rows): base 1.45 / 2.0, relay 1.63 / 3.23 DO units; rows 44 / 48 vs 88 / 151 | -| C6 | bundle raw / gzip KiB; startup ms | 2,537.10 / 610.56 KiB at base SHA (dry run, K3) | 2,652.13 / 636.21 KiB at `276c16c` (w0930 deploy; the same bundle serves every flag) | same bundle | **measured, one deploy each.** Full run (C6-relay, deployed `45f3705`): 2,652.13 / 636.21 KiB, startup relay worker 15 ms, base worker 12 ms. Startup: base 10 ms, relay 8 ms, strict 13 ms (w0930); v0930: 11 / 15 ms. The spike adds +115 KiB raw / +25.7 KiB gzip, of which patch 0003 is +3,358 B raw / +379 B gzip. Free limit: 3 MiB compressed. v1 relay: 2,757.68 / 651.81 KiB. | - -### 8.3 Behaviour - -| ID | Pass criteria | Base | Relay | Strict | -|---|---|---|---|---| -| B1 | socket survives hibernation and restart | **lost**: after 150 s idle the runtime epoch changed and all 4 sockets closed 1008 "socket authority mismatch"; warm 298 ms (n = 1 run) | **survived** idle + 2 restarts: warm 130, afterIdle 118, second 3,633, afterRestart 1,591, afterRestart2 71.9 ms (n = 1 run) | — | -| B2 | catch-up after 50 / 5,000 edits: time, bytes, rows scanned, text equal | pass, text equal; 50-edit p50 / p90 363 / 373 ms; 5,000-edit 394 ms, 108,892 B (n = 50 × 10 + 1) | pass, text equal; 366 / 388 ms; 5,000-edit 429 ms, 100,287 B (n = 50 × 10 + 1). Rows scanned: not measured (not recorded by the harness) | — | -| B3 | bootstrap of 100 notes (20 relay-edited) | pass, all match; p50 / p90 1,713 / 2,394 ms (n = 3) | pass, all match; 1,936 / 2,243 ms (n = 3) | — | -| B4 | revocation mid-stream: 4403 ≤ 1 s, no append after the fence | pass: 4403 after 182 ms, 0 appends after (n = 1) | pass: 4403 after 164 ms, 0 appends after (n = 1) | — | -| B5 | reset race: one install, zero lost edits | n/a | **measured: 6/6 pass**, winners A, B, A, B, A, B; race p50 1,813 ms, p90 2,097 ms (`B5-2026-09-29T20-16-32-720Z.json`, yaos-relay2-reset, e1abb36). Full-n (`B5-relay`, n = 20 races): **20/20 pass**; race p50 / p90 / p99 1,384 / 1,693 / 1,964 ms; winners alternate A/B; the loser gets lease-denied `held`, then 4409, and rebases onto epoch 2; all 14 checks ok (one install, zero lost edits, GET hash matches). Uploads (n = 5 each, all installed): 100k 315 ms, 1m 958 ms, 5m 4,081 ms (request 6,765,332 B) | — | -| B6 | 3× resend → one append, same receipt | **skipped by design**: base idempotence lives in the HTTP candidate path, and the relay candidate adapter does not apply | pass: appendsDelta 1, dedupeHits 3, same receipt; a reused candidate id with different bytes is rejected | — | -| B7 | 5 MiB/s flooder → 1013; victim L2 within 20% | pass: no rate limit, flood achieved 5.01 MiB/s with no close; bystander p50 306 → 321 ms (+5.0%), absorbed by the 250 ms debounce | **pass**: our 1013 "relay rate limit" at 26.9 s; achieved 0.10 MiB/s; bystander 63.2 → 62.4 ms (−1.3%) | pass: 1013 "relay rate limit" at 26.3 s; 0.26 MiB/s; bystander 57.5 → 57.4 ms (−0.3%) | -| B8 | delete with sockets open | pass: delete 666 ms; sockets 1008 "body deleted" +214 ms; reopen refused 409; GET 404 | pass: delete 673 ms; same close / 409 / 404 behaviour | — | -| CW | invariant #7 under concurrent writers | pass: 121/121 hash claims hold (2,372 edits; smoke 21/21) | pass: **119/119** hold, 104 materialised, 15 known (2,375 edits; smoke 21/21) | — | - -### 8.4 Limits - -| ID | Metric | Base | Relay | Strict | -|---|---|---|---|---| -| X1 | max concurrent body sockets (100 → 2,000) | **32**: 429 `body_socket_limit` at #33 (75 attempted, 43 failed); probe 307 / 315 ms | **2,000/2,000**, 0 failures (highest step tested; no limit found). Probe p50 / p90: 69.8 / 253 at 100, 89 / 108 at 250, 77 / 86.5 at 500, 101 / 201 at 1,000, 106 / 232 at 2,000 ms; linear memory 2.36 MB; open-all 94.7 s at 2,000 | — | -| X2 | max sustained append rate, 1 body / 10 bodies (10 s steps; limit = propagation p50 > 2× floor) [serial-after] | criterion n/a: the 250 ms debounce keeps p50 above 2× floor (195–312 ms vs floor 50.8), so the harness reports "max 0" (artifact). Sustained 100/s on 1 and 10 bodies, 0 lost, 0 closes (p50 230 / 225 ms) | **≥ 400/s** on 1 and 10 bodies (p50 86.6 / 87.7 ms, floor 47.5). At 800/s (highest step tested): p50 122 / 113 ms, 0 loss, 0 closes | **200/s**. At 400/s on 1 body: platform 1013 "Service overloaded", p50 3,142 ms, 1 reconnect, 1,520 frames resent; 10 bodies p50 304 ms (floor 61) | -| X3 | largest note for step2 and checkpoint (1 / 5 / 10 MB) | 1 MB ok (open ≈ 755 ms, compact 946 ms, 200); **5 MB compact HTTP 500** (164 ms). 10 MB: not measured (seed only) | 1 / 5 MB ok: open 688 / 1,124 ms; edit ack 82 / 72 ms; compact 786 / 846 ms (200). 10 MB: not measured (seed only). The convergence "fail" is a **checker artifact** (section 8.7) | — | -| X4 | catch-up of 100 bodies × 50 edits stale | ok: HTTP catch-up 437 ms, 608,662 B; socket reopen 9,313 ms, 226,992 B (A3: 676 ms) | ok: HTTP catch-up 385 ms, 536,753 B; socket reopen 9,446 ms, 157,678 B | — | - -### 8.5 Compaction - -| ID | Metric | Value | -|---|---|---| -| K1 | alarm merge at tails of 50 / 500 / 5,000: ms, memory, rows before and after | **Compact (measured, n = 3 per tail, [B∥10]):** base 362 / 556 / 555 ms; relay 220 / 213 / 226 ms, log rows go to 0 afterwards. **Alarm (`K1-alarm-relay`): inconclusive** — the alarm had already checkpointed during the appends (lastCheckpointMs 0 at < 1 ms resolution; log rows 14–21 before = after; 0 checkpoint rows written). Real alarm cost is taken from C1/C2 alarm CPU instead: p99 59 ms (C1), 108 ms (C2-quick), 1,041 ms (C2-stress, 50k history). Memory per alarm: not measured. Smoke (vault-wide compact, small n): relay 193–368 ms for tails of 50–2,000, base 515–559 ms. Local: `checkpointsFromCache` 3.3 → 1.8 ms per pass for a 30 KiB body. | -| K2 | reset build, desktop lease path p50 / p90 (measured, n = 10, M4 Pro, Node v26.5, fd38e71) | 100k: 12.1 / 12.8 ms. 1m: 25.9 / 26.3 ms. 5m: 97.7 / 104 ms. stress: 13.6 / 14.2 ms. Duty-cycle worker (measured): 1m at duty 25 is 166 / 205 ms and at duty 17 is 469 / 605 ms; 5m at duty 25 is 610 / 1,052 ms and at duty 17 is 1,742 / 2,667 ms. Mobile ×4–6 (**inferred**): 1m 103–155 ms, 5m 391–586 ms. Upload ≈ live size + 120 B. 5m peak heap 176 MiB. A 5m server-side reset traps (`prepareSemanticReset`, 96 MiB cap). | -| K2-upload | deployed reset upload | Pre-round-2 JSON (B5, measured): 100k p50 395 ms (n = 5, lease p50 284 ms, 134,071 B); 1m p50 1.08 s; 5m 5.0 → 20.9 s, then `lease_expired` at 115.7 s. Round-2+ binary (core3-live-reset-5mb, coresmoke-1, 5.24 MB, n = 4, all 200): 1,740 / 3,114 / 1,839 / 3,272 ms; the GET after reset takes 1.5–3.0 s with `hashState: known`; HEAD 192–269 ms; cooldown lease 429 with `retry-after: 1`. Round-2 core2 (n = 6): 3.7, 3.3, 3.6, 1.7, 3.1, 3.1 s. | -| K3 | stateless byte ops (c) vs transient doc (a) vs JS yjs (b) | **Measured, Node, not workerd** (K3.md). Merge: (c) is ≈ 2.5× faster than yjs and super-linear (50k frames: 4.4 s vs 12.8 s). SV/diff (c): 0.26 ms quick, 1.2 ms 1 MB, 6.7 ms 5 MB, 14 ms 10 MB. Wasm high-water ≈ 4–5× merged size for merge (10 MB → 48 MiB); ≈ 20 MB merged reaches the cap. Memory never shrinks, and does not grow on repeat. Lone-surrogate loss occurs only on the ywasm Doc path. Bundle: 2,537.10 → 2,541.45 KiB raw, 610.56 → 611.11 KiB gzip; Wasm is ≈ 340 KB of the gzip. | - -### 8.6 v1 projections: did they hold? - -| Projection (v1, A5) | v1 evidence | v2 measured | Held? | -|---|---|---|---| -| 10–50× less memory | 32 × 512 KiB: 37.5 MB base vs 1.11 MB relay | C3: 37.22 MB vs 4.06 MB = **9.2×** (32 resident vs 0) | **Nearly**: just under the 10× floor (v1 measured ≈ 34×). The cause of the higher v2 figure was not isolated | -| 30–100× more concurrent notes | 100/100 vs 429 at #33 | X1: 2,000/2,000 vs 32 = **≥ 62×** (no ceiling found) | **Yes** (lower bound) | -| 3–5× faster propagation | 300 → 52 ms | L2: 308 / 66.0 ms = **4.67×** p50 (strict 5.48×), p99 608 → 314 | **Yes** at p50 and p99 on L2. L4-relay p99 is worse than base (2,305 vs 543, open) | -| 30–100× less CPU on heavy notes | 19 ms → < 1 ms | C1 WS CPU per invocation p99 64.3 → 4.19 ms = **15×**; p50 1.36 → 0.22 ms = 6.3×. Per-keystroke heavy-only CPU: not measured separately (gql is whole-phase, R6) | **Partly**: 6–15×, not 30–100×. CPU moves into the checkpoint alarm (p99 up to 1.04 s) | -| Near-zero idle cost | hibernation survives | B1-relay survives 150 s idle + 2 restarts (base: 1008 on all sockets). Idle DO requests/day: inferred 1,056 (1%) in both modes; idle DO requests were not measured directly (C5 measures bursts and catch-ups) | **Yes**: idle cost equal to base and ≈ 1%; relay avoids the post-eviction reconnect | - -### 8.7 Convergence - -For each scenario, A, B and a fresh C have identical text and state vectors, the server GET equals the client text, -and the recorded hash equals the client's canonical hash. Summaries: `convergence-suite.json` (relay), -`convergence-suite-strict.json` and `convergence-suite-base.json`. - -| Scenario | Smoke (v0930) | Full n relay | Full n strict | Full n base | -|---|---|---|---|---| -| L2 | pass | **pass** | pass | pass | -| L4 | pass | **pass** | pass | pass | -| quick trace | — | not in suite (accounting PASS) | not in suite (accounting PASS) | not in suite | -| stress trace | pass | **pass** (50k) | pass (50k) | **FAIL**: base semantic reset (4409), see below | -| B2 | pass | **pass** | — | pass | -| B3 | pass | **pass** | — | pass | -| B5 | pass (6/6, component) | **pass 20/20** | — | n/a | -| B6 | pass | **pass** | — | no block (skipped by design) | -| X1 sample | pass | **pass** | — | pass | -| CW (#7) | pass 21/21 | **pass 119/119** | — | pass 121/121 | - -Totals (measured): relay **9/9 pass**; strict 3/3 run, all pass; base 6 pass, 1 fail, 1 n/a, 1 skipped. - -- **C2-stress-base FAIL is base hitting its own semantic reset under stress, which the raw harness cannot follow. It - is not relay data loss.** Server-side semantic compaction bumped the body to epoch 2 at frame 10,226 and sent 4409 - to all 6 sockets; 165 frames were outstanding at drain (GET 210,869 B vs client 210,844 B). Local yjs census at - frame 10,226 (inferred): 20,478 structs, deleted ratio 0.311, which crosses the soft deleted-ratio threshold - (≥ 0.25 with ≥ 10k structs, projected reduction ≥ 0.40). Ruled out: size limits, hard limits, the ywasm cap - (13.1 MB of 96 MiB). The real client rebases on 4409; the raw harness client does not reconnect. Relay and strict - have no server-side doc and no server-initiated reset, and converged at 50k. -- **C3-base** (outside the suite): a fresh C cannot open a 33rd socket. This is the real base limit. -- **X3-relay "fail" is a checker artifact** (outside the suite): text and SV are equal on A, B, fresh C and GET. A - 5 MB body gets no hash claim by design (> 256 KiB), and `checks.ts` treats the empty-string hash header as - non-null, which produces a false inv7 "VIOLATED". Harness only; not fixed. - -Frame accounting (4.3): **PASS in all 16** relay/strict runs with diagnostics (measured): C2-quick relay/strict -(5,000), C2-stress relay/strict (50,000), C4 relay/strict (50), L4 relay/strict (5,000), MB lean ×6 + strict ×2. -Base n/a. Invariant evidence at full n: 4 (B4 0 appends after the fence), 5/11 (B5 20/20, one install per race), 7 -(CW-relay 119/119, CW-base 121/121, inv7 pass in every convergence leg), 8 (B6 dedupeHits 3, appendsDelta 1), 10 (K1 -compact → 0 log rows; C2-stress converges at 50k). - -### 8.8 Relay v3 (group commit) - -All values are measured on deployed workers (runs r3-1002/b/c). Full tables are in -`experiments/results/relay3/RESULTS.md`. Latency phases are [conc]. - -| Area | Result | -|---|---| -| Rows/edit, MB type5 gql | base 5.58, v2 5.63, v3 0.42, **v3nc 0.29**, v3b5 1.53 | -| Rows/edit, C4 counter (spaced) | v2 5.0, v3 3.0, v3nc 2.0, v3b5 3.0. Phase-level gql: base 10.0, v2 8.5, v3 4.18, v3nc 3.18 | -| L2 propagation p50 / p99 (n = 290) | base 299 / 696, v2 63.0 / 396, **v3 47.7 / 76.3**, v3b5 301 / 333 | -| L2 receipt (origin ack) p50 / p99 | v2 62.9 / 396, **v3 350 / 482**, v3b5 604 / 926 | -| L4 per-frame p50 / p99 (n = 4,990) | v3 47.2 / 231 (f1001 v2 73.7 / 2,305) | -| L5 edit → cleared, real `VaultSync` (n = 90) | base prod 479, v2 relay 67.0, **v3 native 353** (p99 1,177), v3 native250 603. 8-edit burst first / last 1,246 / 359. 0 HTTP POSTs | -| MB type5 receipt p50 / p99 | v3 907 / 1,642, v3nc 949 / 1,603 (max-wait bound) | -| CRASH (pending buffer dropped) | online 5/5, offline-origin via peer 5/5. 51 frames dropped, **0 lost**, recovery p50 ≈ 0.8 s, fresh C converges | -| FENCE (cumulative acks) | too-large 5/5. Rate 6/6 refused rounds: 0 acked or durable after the refusal | -| Convergence suite | 9/9 (L2, L4, C2-stress, B2, B3, B5 20/20, B6, X1, CW 121/121) | -| B1 / B6 / B8 / X4 | pass / pass / pass / pass | -| B4 | invariant 4 holds; **convergence FAIL** (R11) | -| B7 | **1/3 pass** (R12) | -| Autosave 1 s, open note | v3nc 2.15 rows/save (gql), one commit per save; prop p50 45 ms | -| Autosave, closed note (HTTP candidate every 5 s) | 13.82 rows/POST (gql phase-level), POST p50 297 ms. After 14e5250: **3 rows/POST, ≈ 4.0 amortized with the coalesced catalog event (inferred, local row accounting; not deployed)** | -| Canvas identical re-save | 0 rows, no POST. 1 GET `semantic//state` (`canvasManager.ts:415/417`), code | - -## 9. Cost model - -Workers Free limits were checked on 2026-09-30 in the Cloudflare docs -([DO pricing](https://developers.cloudflare.com/durable-objects/platform/pricing/), -[Workers pricing](https://developers.cloudflare.com/workers/platform/pricing/), -[DO limits](https://developers.cloudflare.com/durable-objects/platform/limits/)). - -| Limit (Workers Free) | Value | Note | -|---|---|---| -| DO requests | 100,000 / day | Counts HTTP requests, RPC sessions, WebSocket messages and alarm invocations. Incoming WS messages are billed at **20:1**; outgoing messages and protocol-level pings are free. Exceeding any free limit makes further operations of that type fail until 00:00 UTC. | -| DO duration | 13,000 GB-s / day | Hibernatable idle is not billed. | -| SQLite rows written | 100,000 / day | **Index writes, deletes and `setAlarm()` each count as rows written.** | -| SQLite rows read | 5,000,000 / day | | -| SQLite storage | 5 GB total (1 GB per object on Free, per the FAQ) | | -| DO CPU per request | 30 s default (same table for both plans); resets per incoming request or WS message | The Workers Free Worker limit is 10 ms CPU per request; that applies to the Worker entry, not the DO. The enterprise account differs only in configurable DO `cpu_ms`. | -| Memory | 128 MB per isolate (JS heap + Wasm) | The 96 MiB ywasm cap sits inside this. | -| Worker requests | 100,000 / day | HTTP and WS upgrades; WS messages are not Worker requests. | -| Max SQL row / BLOB | 2 MB | This is why `MAX_DURABLE_UPDATE_BYTES` is 1.75 MB. | - -The calculator is `scripts/relay2/costmodel.py`. Every input is a named parameter with a provenance tag, and it -reruns with measured values (`--set name=value` or `--json`). It was re-run on 2026-10-01 with the full-run (f1001) -inputs below. Its docstring still says the micro-batch maximum is 50, which is stale; the cap is 250. - -The model does not cover two things: -- the lean coalescing pass (runs A, B and E use whole-phase gql rows, which already include it) -- CPU (taken from gql directly, below) - -Every output below is **inferred** from **measured** inputs. Model: 3 devices, 4 sockets each, 60 s pings, 270 s -ticket refresh, 2 h editing per day at 25% typing duty, 5 frames per typing second, Free limits (100k DO -requests/day, 100k rows/day). - -**Inputs** (measured, f1001): -- `base_rows_per_flush` = 3.10 rows/edit (MB/C2 gql) × 9,000 / 7,200 flushes = **3.875**. -- Runs A, B and E use whole-phase gql rows/edit (MB-lean-mb10-stream 5.92, MB-strict-stream 11.84). These already - include checkpoint, floor, coalescing and candidate rows, so A and B set `checkpoint_fixed_rows=0` and - `journal_delete_rows_per_entry=0`. E keeps the modelled checkpoint rows as well (double-counted, pessimistic) and - uses base 3.10 per flush. -- Runs C and D use the C4 relay counter (append rows only: relay 5.00, strict 11.00) plus the modelled checkpoint - rows (8 fixed + 2 per pruned journal row). - -**Output** (`costmodel.py --set …`): - -| Run | Inputs | Typical base rows/day | Typical relay rows/day | Typical + 2,000-note import base / relay | DO req/day typical base / relay | Typing h/day to 100k rows base / relay | -|---|---|---|---|---|---|---| -| A relay (gql) | relay 5.92, mb 10; base 3.875/flush; ckpt rows included (0/0) | 30,024 (30%) | **53,460 (53%)** | 88% / **115%** | 2% / 3% | 1.67 / 0.94 | -| B strict (gql) | relay 11.84, mb 0 | 30,024 (30%) | **106,740 (107%)** | 88% / 180% | 2% / 3% | 1.67 / 0.47 | -| C relay (C4 counter + modelled ckpt) | relay 5.0, mb 10 | 45,576 (46%) | **64,620 (65%)** | 103% / 125% | 2% / 3% | 1.10 / 0.77 | -| D strict (C4 counter + modelled ckpt) | relay 11.0, mb 0 | 45,576 (46%) | **118,620 (119%)** | 103% / 191% | 2% / 3% | 1.10 / 0.42 | -| E relay (gql, ckpt double-counted, pessimistic) | relay 5.92, base 3.10/flush, mb 10 | 39,996 (40%) | 72,900 (73%) | 96% / 135% | 2% / 3% | 1.25 / 0.69 | - -Run A output (verbatim): - -``` -| Day shape | Mode | DO req/day | % free | Worker req/day | % free | Rows written/day | % free | Notes | -|---|---|---:|---:|---:|---:|---:|---:|---| -| idle | base | 1,056 | 1% | 0 | 0% | 0 | 0% | 12 sockets, 21,120 WS msgs | -| idle | relay | 1,056 | 1% | 0 | 0% | 0 | 0% | 12 sockets, 21,120 WS msgs | -| typical | base | 2,300 | 2% | 342 | 0% | 30,024 | 30% | 9,000 frames, 7,200 appends, 180 POSTs, 144 ckpts | -| typical | relay | 2,648 | 3% | 60 | 0% | 53,460 | 53% | 9,000 frames, 9,000 appends, 0 POSTs, 180 ckpts | -| typical+import | base | 5,000 | 5% | 2,742 | 3% | 87,774 | 88% | +2,000 notes | -| typical+import | relay | 5,648 | 6% | 2,460 | 2% | 115,300 | 115% | +2,000 notes | -``` - -**CPU** (measured, gql per-invocation; not in the model). The 10 ms limit applies to the Worker entry; a DO request -gets 30 s CPU on Free. 10 ms is still the bar for a cheap body edit. Per WS message, p99: - -| | base | relay | strict | -|---|---|---|---| -| C2-quick WS p99 (n = 5,000) | **30.7 ms (over)** | 4.6 ms | 5.7 ms | -| C1 WS p99 (5,101 / 10,188 invocations) | **64.3 ms (over)** | 4.2 ms | 5.7 ms | -| C2-stress WS p99 | 7.3 ms (10,226 frames, provisional) | 2.8 ms (50k) | 1.9 ms (50k) | -| checkpoint alarm p99 | n/a | 59 ms (C1) / 108 ms (C2-quick) / **1,041 ms (C2-stress)** | 98 / 102 / 817 ms | - -`exceededCpu` = 0 in every phase. K3 shows an exact SV or diff on a 5–10 MB body costs 7–14 ms, which is why bodies -over 256 KiB use the incremental SV (R4). - -Reading the model: -- **Relay with lean rows fits Free on a typical day at 53–73%**, vs base at 30–46% (≈ 1.7× base rows). **Strict does - not fit** (107–119%), so lean rows are a requirement. -- **Rows written, not requests, is the binding free-tier limit.** Typing hours/day to reach 100k rows: relay - 0.69–0.94 h vs base 1.10–1.67 h. -- **Import days.** A typical day plus a 2,000-note import exceeds 100k rows for the relay (115–135%); base sits at the - edge (88–103%). The import is dominated by lifecycle rows (`import_rows_per_note`, assumed 25). A large import is a - two-day operation on Free in either mode. -- **DO requests are not binding**: 2–3% on a typical day (≤ 6% with import), 1% idle in both modes. Worker - requests/day: base 342, relay 60 (no candidate POSTs). Typical-day relay DO requests are 1.15× base (2,648 vs 2,300), - which strictly exceeds reversal condition 3's 1.1× bar while both sit at 2–3% of Free (section 13). -- **Micro-batching.** At 5 frames/s, mb 10 does not merge appends (MB: 5.54 vs 5.55 rows/edit). Its value is the - latency tail (stream p99 316 vs 416). Only mb ≥ 200–250 coalesces at typing cadence, and it gives back much of the - latency win (smoke mb250 p50 ≈ 190–300 ms). -- **CPU does not bind for the relay**: WS p99 ≤ 5.7 ms and alarms ≤ 1.04 s, under the 30 s DO limit. The alarm is a - wall-clock stall for the output gate, though (R9). Base already exceeds 10 ms per WS message. -- **Spend.** The runs used an unlimited enterprise account, profiled against Free. No quota was hit apart from the - 500-DO-namespace account cap (test-environment note, section 12). - -### 9.1 Relay v3 cost model - -Calculator: `scripts/relay2/costmodel3.py`. It imports `costmodel.py`, uses the same Free limits and adds the heavy -and autosave shapes. All values are **inferred** from the measured r3 gql inputs in -`experiments/logs/relay3/costmodel3-inputs.json`. A second run with the f1001 base 3.10 and relay 5.92 changes -only the base and v2 rows (base 30%, v2 53% typical); see `RESULTS.md` section 7. - -| Day shape | Config | DO req/day | % Free | x base | Worker req/day | Rows written/day | % Free | x base | Notes | -|---|---|---:|---:|---:|---:|---:|---:|---:|---| -| typical | base | 2,300 | 2.3% | 1.00 | 342 | 52,344 | 52% | 1.00 | 9,000 frames, 7,200 appends, 180 POSTs, 144 ckpts | -| typical | relay v2 | 2,648 | 2.6% | 1.15 | 60 | 50,850 | 51% | 0.97 | 9,000 frames, 9,000 appends, 0 POSTs, 180 ckpts | -| typical | v3 | 2,648 | 2.6% | 1.15 | 60 | 2,790 | 3% | 0.05 | 9,000 frames, 9,000 appends, 0 POSTs, 180 ckpts | -| typical | v3 (candidateId) | 2,648 | 2.6% | 1.15 | 60 | 3,960 | 4% | 0.08 | 9,000 frames, 9,000 appends, 0 POSTs, 180 ckpts | -| typical | v3+B5 | 2,198 | 2.2% | 0.96 | 60 | 13,950 | 14% | 0.27 | 9,000 frames, 9,000 appends, 0 POSTs, 180 ckpts | -| heavy | base | 8,132 | 8.1% | 1.00 | 1,314 | 335,002 | 335% | 1.00 | 57,600 frames, 46,080 appends, 1,152 POSTs, 922 ckpts | -| heavy | relay v2 | 10,910 | 10.9% | 1.34 | 60 | 325,440 | 325% | 0.97 | 57,600 frames, 57,600 appends, 0 POSTs, 1,152 ckpts | -| heavy | v3 | 10,910 | 10.9% | 1.34 | 60 | 17,856 | 18% | 0.05 | 57,600 frames, 57,600 appends, 0 POSTs, 1,152 ckpts | -| heavy | v3 (candidateId) | 10,910 | 10.9% | 1.34 | 60 | 25,344 | 25% | 0.08 | 57,600 frames, 57,600 appends, 0 POSTs, 1,152 ckpts | -| heavy | v3+B5 | 8,030 | 8.0% | 0.99 | 60 | 89,280 | 89% | 0.27 | 57,600 frames, 57,600 appends, 0 POSTs, 1,152 ckpts | -| autosave-8h (open note) | base | 32,540 | 32.5% | 1.00 | 29,142 | 480,744 | 481% | 1.00 | inferred: 1 flush + 1 candidate POST per save | -| autosave-8h (open note) | v3 | 5,528 | 5.5% | 0.17 | 60 | 64,710 | 65% | 0.13 | open note: 1 socket frame per save (MB autosave gql) | -| autosave-8h (closed note) | v3 | 8,408 | 8.4% | 0.26 | 5,820 | 82,393 | 82% | 0.17 | closed note: 1 HTTP candidate per 5 s (HTTPSAVE gql) | - -| Config | Rows per typing-second | Non-stop typing hours/day to 100k rows (excl. idle) | -|---|---:|---:| -| base | 29.0 | 0.96 | -| relay v2 | 28.1 | 0.99 | -| v3 | 1.4 | 19.16 | -| v3 (candidateId) | 2.1 | 13.23 | -| v3+B5 | 7.7 | 3.63 | - -Inputs: base_rows_per_edit=5.58, relay_rows_per_edit=5.63, v3_rows_per_edit=0.29, v3c_rows_per_edit=0.42, v3b5_rows_per_edit=1.53, v3b5_frames_per_edit=0.5, autosave_rows_per_save=2.15, httpsave_rows_per_post=13.82, base_autosave_rows_per_save=14.875 - -- **Rows are no longer the binding limit.** v3 is 3% typical and 18% heavy, against v2 51–53% and 325–342%. 100k - rows takes 19.2 h of non-stop typing. -- **The autosave plugin day fits:** 65% (open note) and 82% (closed note, HTTP path). Base, inferred, is ≈ 460–480%. -- **HTTP path after 14e5250 (inferred, not deployed; the table above still uses the measured 13.82).** The closed-note - POST drops to 3 rows (≈ 4.0 amortized with the alarm-coalesced catalog event). Scaling the measured 82,393 rows by - 4.0 / 13.82 gives ≈ 24k rows, about 24% of Free for autosave-8h closed note. This is arithmetic only; re-measure - HTTPSAVE before relying on it. -- **Typing rhythm (local accounting, inferred; added with 4a34f56).** The 0.29 rows/edit input is a best case: - steady typing at 5 keys/s, where most commits happen on max-wait. Before the commit-rate cap, any gap over 300 ms - committed, so typing at 1–3 keys/s cost one commit per keystroke: 3.00 rows/key with candidate ids, about 2 without. - Applying that to the heavy shape (57,600 frames, 2 keys/s for 8 h) would give ≈ 115k–173k rows, over Free. With - the cap (idle commits ≥ 1 s apart per body), local virtual-time accounting gives 1.55 rows/key at 2 keys/s, - 1.03 at 3, 0.38 at 5, 0.25 at 8 and 2.23 for bursty 100–2000 ms gaps (candidate ids). That bounds a body at - 3 rows per second of activity. The heavy shape at 2 keys/s becomes ≈ 89k rows with candidate ids - (arithmetic, not measured). The rows/edit inputs above should be re-measured at 1–3 keys/s before relying on - the 18% heavy figure. -- **DO requests are not changed by v3** (1.15× base typical, 1.34× heavy, both ≤ 11% of Free). R10 stands as in - v2. B5 would bring DO requests to 0.96–0.99× base, but at 4.9× the rows (R13). - -## 10. E2EE compatibility - -Goal: relay mode should need plaintext CRDT structure only in removable, optional server paths. The table lists every -relay-mode server operation that reads update bytes as more than opaque blobs. - -| Operation | Where | Needs | Moves to clients how | -|---|---|---|---| -| **Checkpoint byte merge** (`mergeUpdatesV1`) | alarm → `checkpointBody` → `durableMergedBytes` | Plaintext CRDT structure (it parses structs and IDs) | A client under a lease uploads an encrypted snapshot covering sequence S. The server stores it as an opaque checkpoint and prunes the journal ≤ S (the same row shape as reset without the epoch bump). Mobile-only is fine because the lease is available to any device. | -| **Lazy hash materialisation** (`reconstructDocument` → `readText` → sha256) | `bodyHttpState` (GET, catch-up) when the catalog hash is NULL | Plaintext **text** (a full ywasm Doc) | Delete it. The hash comes only from client claims (the envelope). Unknown stays unknown, and HEAD already exposes `hashState`. Under E2EE the hash itself must be a keyed hash (HMAC with a vault key) so it leaks nothing. | -| **Head SV tracking** (`stateVectorFromUpdate`, `maxStateVector`) | every append (`commitFrames`), rebuild | Plaintext structure (IDs and clocks) | The client sends its post-update SV in the envelope (it already does for D6). Under E2EE the SV is metadata the client chooses to reveal. Either accept the SV leak (client IDs and clock counts, no content) or have the server track only sequences, with step1 answered by "all rows after the client's last seen sequence". | -| **Step2 diff** (`diffUpdateV1(merged, clientSV)`) | step1 reply | Plaintext structure | Replace SV-based diff with sequence-based catch-up: the client sends its last durable sequence and the server returns checkpoint + rows > seq as opaque blobs, which is already how `replyStep2Unmerged` works. Costs more bytes; no crypto change. | -| **Growth-cap no-op test** (exact merge equality) | `commitFrames` ≤ 256 KiB | Plaintext structure | Drop it and rely on client idempotence: receipts via candidateId, and clients skip resends whose SV is already covered by the last acked SV. The large-body identical-bytes test works on ciphertext only if encryption is deterministic, which it should not be. | -| **Micro-batch merge** (`mergeUpdates(live…)`) | `commitFrames` when batch > 1 | Plaintext structure | Store the frames as separate journal rows in one transaction (N journal rows, one sequence each) or disable micro-batching. | -| **Bootstrap / catch-up / GET byte merge** | `durableMergedBytes` | Plaintext structure | Serve checkpoint + tail parts unmerged (opaque); the client merges. | -| **Merged-bytes cache** | `MergedEntry` | Opaque storage of plaintext merged state | Disappears with the merges. | -| **`syncDocumentCache`** (since round 3 it only discards or marks a resident base doc stale; no apply) | after append | None now (no ywasm call) | Delete it together with the base body path (section 11). | -| **Reset structural check** (`snapshotStructurallyValid`: SV parse) | reset route | Plaintext structure (SV parse only) | Replace with a size check plus a client-signed header. | -| **Recovery projection** (snapshot format 4, b3 P2) | vault wake alarm → `runStateProjectionPass`; capture fallback `materializeCaptureContent` | **Opaque bytes.** The stored checkpoint, journal, and tail bytes are written to R2 unchanged as a content-addressed state object. Yjs is not decoded on the server. | Done: the RecoveryJob no longer decodes, and clients decode and verify size and sha256 on restore. Under E2EE the stored bytes are ciphertext and the key hash must be keyed (see lazy hash). A wrong claimed hash is detected only on the client. | -| **Semantic reset** | client builds | None on the server (opaque install); the server trusts `coveredSequence` | Already on the client. | -| Frame size, rate, epochs, authority, receipts, sequence, attribution digests | throughout | **Opaque bytes** (length and sha256 of ciphertext) | No change. | - -Classification summary: -- Opaque today: size, rate, digest, receipts, epochs, sequence, lease/reset CAS. -- Needs structure: merge, SV, diff, growth cap, micro-batch merge, reset SV parse. -- Needs text: the lazy hash only. - -The brief's expectation was that only the optional checkpoint and the lazy hash need plaintext. That understates it: -SV tracking and step2 diff also need structure. Each has a removal path (the envelope SV and sequence-based catch-up) -that works today with `replyStep2Unmerged`. - -## 11. Migration plan - -**Cutover policy (confirmed).** `docs/operations.md` says: "There is no in-place migration from any earlier schema or -storage format … A release changing any pin or required class must declare another fresh or guided cutover." - -The relay keeps storage format 4, schema 8 and protocol 5. It adds one lazily created table and one capability key. -- A flag flip on an existing deployment is technically possible, because the tables mean the same thing. -- Receipts and sockets differ by mode, so the recommendation is: **flag changes only on a fresh deployment**, paired - with a protocol-capability check in the client (`relayBodies: 2`). -- Rollback (relay → base) on the same storage is possible, because base can read relay-written journal, checkpoint - and catalog rows, which have the same shape. It is untested and is open question Q6. - -**Phases:** - -| Phase | Flag / gate | Contents | Exit criteria | -|---|---|---|---| -| 0 | none | Done in the spike: G1–G8, G11, G13–G17, G19, G20 (round 3); no silent drops, lean rows, `checkpointsFromCache` (round 4). Left: G9/R2 decision, G10 mitigations, G18 residual, the lean-off coalescing step | unit tests + flag-off 0 new failures (round 4: 3,635 pass) | -| 1 | `YAOS_RELAY_BODIES=true` + `YAOS_RELAY_LEAN_ROWS=true` + `YAOS_RELAY_MICROBATCH_MS=10`, opt-in deploy vars | Server relay plus the unmodified client (unenveloped; hash via lazy path ≤ 3 MiB; HTTP candidates still clear receipts) | full-n section 8 tables; section 13 not triggered | -| 2 | client capability `relayBodies ≥ 2` | Envelope + socket-ack receipts in `VaultSync` (fencedWebSocket `send()` wrapper; candidate observer → clientFrameId/candidateId); HTTP candidate POST becomes fallback | L5 relay ≤ base; candidate clears via socket in ≥ 99% | -| 3 | client | Lease-based semantic reset in the plugin (policy + builder from `scripts/relay2/reset`), receiving-side validation (owned elsewhere) | B5 at full n; mobile-class reset in budget | -| 4 | default on, fresh-install release | Delete the base body path (below) | one release of relay-default without reversal | -| 5 (E2EE prep) | separate | Client checkpoints, sequence-based catch-up, remove the lazy hash (section 10) | | - -**Code that becomes dead in phase 4** (`wc -l` on the working tree, 2026-09-30; relay line counts 2026-10-01). - -The root document, Canvas semantic documents and the creation-candidate fence stay on the base path, so most files -are only partly deletable. - -| File | Lines | Deletable part | Estimate | -|---|---:|---|---:| -| `server/src/semanticCompaction.ts` + `…Policy.ts` + `…Runtime.ts` | 289 + 244 + 432 = 965 | Body share (Canvas keeps server compaction unless it also moves) | 350–600 | -| `server/src/vaultDocumentCache.ts` | 1,178 | Body validation mirror (`validateBodyUpdate`, `stage/discardValidatedBodyUpdate`, `rebuildValidation`), body admission and capacity (`admitBody`, `ensureBodyCapacity`, `cleanBodyCandidates`, `loadedBody*`) | ≈ 250–300 | -| `server/src/vaultSocketService.ts` | 1,372 | `bodyUpdateAdmissionError` (23) + body branch of `handleSyncFrame` validation and staging + base body step1 load | ≈ 120–180 | -| `server/src/server.ts` | 1,573 | Body share of `scheduleFlush`/`flushDocument`/`flushLoadedDocuments` (1196–1312, 1472–1476, ≈ 125); body semantic-compaction wiring; body `writeLiveCheckpoint` doc path | ≈ 100–150 (the flush path stays for Canvas) | -| `server/src/vaultCandidateService.ts` | 379 | Ordinary active-body candidate commit (the creation fence stays) | ≈ 150–200 | -| `server/src/crdt/frontmatterSemanticSnapshots.ts` + `frontmatterSemanticValidation.ts` + `shared/frontmatterSemanticValidation.ts` | 98 + 10 + 207 | Moves to receiving clients (the client already shares `shared/`) | 108 server-only | -| `vaultDocumentStore.ts` `reconstructDocument` body uses / `writeCheckpoint` via doc | (2,412) | Only after phase 5 (lazy hash) | ≈ 50 | -| **Total** | | | **≈ 1,100–1,600 server lines**, against +2,235 relay lines at `7feae39` (`relayBodies.ts` 1,249, `relayBodyStore.ts` 536, `relayFlag.ts` 133, `relayRoutes.ts` 137, `ywasmByteOps.ts` 180), plus lean-mode branches in `vaultDocumentStore.ts` | - -Client side: -- `src/sync/vaultSync.ts` candidate HTTP plumbing stays as the fallback. Net client code grows by the envelope, - receipt mapping, reset policy/builder and rebase: `scripts/relay2/reset` gives an upper bound. -- The spike diff at `66fb78c` is 87 files, +16,444 / −56 lines, mostly harness and tests. - -**`packages/server-node` impact.** -- The Node host runs the same `VaultRuntime` compositions, and it already reads `YAOS_RELAY_BODIES` and - `readRelayConfig` (`packages/server-node/src/index.ts:112–114`). -- `ywasmNodeCrdtEngine.ts` (30 lines) exposes the pinned bindings, and the byte ops come from the same patched ywasm - artifact. -- Nothing else changes: storage (`storage.ts`, 678 lines) implements the same SQL port, and `transactionSync` keeps - the append atomic. -- The Node host has no free-tier quota. The rows-per-keystroke concern is SQLite write amplification (fsync per - transaction) and does not block it. - -**Keep ywasm for stateless byte ops, or drop it?** - -| Option | Bundle | Memory | Risk | Verdict | -|---|---|---|---|---| -| Keep ywasm (status quo + patch 0003) | 2,541.45 KiB raw / 611.11 KiB gzip at base + 0003 (K3). The full spike build is 2,652.13 / 636.21 KiB (C6, w0930). Wasm is ≈ 340 KB of the gzip, ≈ 55%. | Wasm high-water ≈ 4–5× merged size, per isolate, until eviction; 96 MiB cap ≈ 20 MB merged | Lone-surrogate loss only on the Doc paths (lazy hash, root/Canvas, reset probe), not on stateless ops | **Required anyway**: the root doc (lifecycle writes root operations, `vaultLifecycleService.ts`) and Canvas semantic docs still need a server CRDT. | -| Drop ywasm for bodies only (JS yjs byte ops) | yjs-only byte-ops ≈ 75 KB raw / 22.5 KB gzip, but it is **added** unless ywasm leaves entirely; `scripts/guard-server-crdt-imports.mjs` forbids `yjs` on the server | JS heap instead of Wasm; merge ≈ 2.5× slower (K3) | Two CRDT implementations on the server | Not worth it | -| Drop ywasm entirely | ≈ −585 KiB gzip | no Wasm cap | Requires root and Canvas to move off server-authored CRDT, beyond this RFC | Out of scope; revisit after E2EE phase 5 | - -Recommendation: keep ywasm, restrict body use to the three stateless ops, and delete body Doc paths in phase 4 and -the lazy hash in phase 5. - -## 12. Open questions and risks - -Design gaps found while checking this RFC against the code are labelled **G#**. Risks are labelled **R#**. - -**Gaps.** G1–G23 were found while checking the RFC against the code in rounds 1–4; G24–G26 come from the full run f1001. -Status is as of `7feae39` (server) and f1001 (measurements). - -| G | Gap (as found) | Status | Fix | -|---|---|---|---| -| G1 | The feed floor never advanced for relay-only workloads, so `vault_journal` grew without bound. | **fixed, round 3** | `runCheckpointPass` advances the floor to `current − 1000`, below every pin. Unit test: 5,000 appends leave 1,001 rows (floor 4002, 41 advances). Full n: K1-compact-relay → 0 log rows; C2-stress-relay converges at 50k (K1-alarm inconclusive). | -| G2 | With micro-batching, authority was checked only at enqueue, so a revocation inside the window still committed. | **fixed, round 3** | Re-validated at flush; dropped frames close 4403 (`authorityDrops`). | -| G3 | Batches were keyed by body only, mixed epochs were fenced together, and reset did not flush pending batches. | **fixed, round 3** | Batches keyed by (bodyId, epoch); reset flushes pending batches before its CAS. | -| G4 | Unenveloped clients make the lazy hash the normal read path (ywasm Doc, plaintext, R1). | **bounded, round 3**; structural until phase 2 | 3 MiB cap plus a 256-body cache keyed by (epoch, head sequence). Above the cap the hash state is `unknown`. | -| G5 | `syncDocumentCache` ran ywasm on the hot path, outside the merge budget. | **fixed, round 3** | It no longer applies updates. Clean residents are discarded; dirty ones are left stale (`residentStaleSkips`). | -| G6 | step1 replies had no authority re-check. | **fixed, round 3** | step1 is authority-checked (4403). | -| G7 | Dedupe ran before the authority check (receipt probe). | **fixed, round 3** | Authority now runs before dedupe. | -| G8 | The incremental SV (> 256 KiB) can overstate state and falsely accept a hash claim. | **fixed, round 3** | `stateVectorExact` flag: bodies over 256 KiB never accept a claim. Large-body reconnect resends are still appended (growth cap only catches identical bytes). | -| G9 | Exact-head reset CAS starvation. | **open** (R2) | Needs a decision. | -| G10 | The server does not verify reset or envelope `contentHash`, so hashes can be poisoned. | **open** | Mitigation options are in section 7. | -| G11 | Duplicate `candidateId` within one micro-batch went undetected. | **fixed, round 3** | Re-ack on the same digest, reject on a different one (`batchDuplicateCandidates`). | -| G12 | The committed `snapshotCoversHead` rejected every lineage-fresh reset. | **fixed, round 2**; docs aligned in round 3 | `snapshotStructurallyValid`. | -| G13 | Pending envelopes are memory-only. | **accepted**, documented as a deviation (round 3) | Hibernation between an envelope and its frame loses the pairing, so the hash is not accepted for that frame. | -| G14 | No `vault_operation_outcomes`, so `/operations/:id/outcome` was blind to relay commits. | **fixed, round 3** | Attribution `operation_id = candidateId` (lean: inline columns). | -| G15 | `diagnostics()` ran `COUNT(*)` over `vault_journal`. | **fixed, round 3** | The count runs only in the diagnostics route. | -| G16 | Multi-origin batches sent `notifyBodyCommitted` to their own origins. | **fixed, round 3** | Every origin is excluded. | -| G17 | Bootstrap docs drifted from code (committed vs WIP). | **fixed, round 3** | Protocol doc aligned with the byte merge. | -| G18 | Write amplification: 9 / 11 CF rows per plain / candidate append, one catalog event per keystroke. | **mitigated, round 4** | Lean rows: 3 / 5 CF rows (measured, n = 20 each); catalog events coalesced. Residual: section 9. | -| G19 | Reset and lease authorized once before a long upload; revocation left lease rows behind. | **fixed, round 3** | Actor re-validated inside lease and reset; leases deleted on revoke and fence. | -| G20 | Over-budget bodies: a 413 inside a batched bootstrap, and an in-memory marker that was lost on eviction. | **fixed, round 3** | Bounded prefix checkpoint, persisted `relay_body_budget` marker, and bootstrap 413 before reserving. Only a client reset recovers the body. | -| G21 | A throw in `relayMessage` sent `VAULT_ERROR` and kept the socket open, which silently lost the frame. | **fixed, round 4** | Pre-append throws close 1011 (`frameErrors`); post-commit throws are counted and never skip fan-out. A frame-accounting identity test covers it. | -| G22 | mb 0 head-of-line tails (0.4–2 s p90) behind the checkpoint alarm. | **mitigated, round 4** | `checkpointsFromCache`, the lean delayed alarm, and mb 5–10 as the default. Full n: L2-strict p99 333, L4-strict p99 866 (max 2,183), MB strict stream p99 506 ms. Relay mb 10 still shows tails (G24). | -| G23 | Lean mode: the raw catalog log lags by ≤ 2 s; frame-0 attribution is pruned at the feed floor; turning lean off needs a coalescing pass first. | **accepted** (round 4) | Documented in `relay2-protocol.md` §6.4. Flag-off migration is not automated. | -| G24 | **Relay p99 tails at full n** (new, f1001): L4-relay p99 2,305 ms / max 3,435 (n = 4,990, [conc]) and C2-quick-relay propagation p99 9,132 ms (n = 5,000, max-rate, [B∥10]). The same config in lane B gave p99 316 (MB-lean-mb10-stream) and L4-strict (serial) 866. | **open** — GO condition 2 | Cause not isolated; tails recur across the run, not cleanly attributable to concurrency. Likely suspect: the checkpoint alarm (G25). Needs a per-frame trace correlated with alarm start/stop. | -| G25 | **Checkpoint-alarm CPU grows with body history** (new, f1001): relay alarm p99 59 ms (C1), 108 ms (C2-quick), **1,041 ms** (C2-stress, 50k edits); strict up to 817 ms. | **open** — GO condition 2 | Under the 30 s DO limit, but a wall-clock stall for the output gate. Candidates: bound the merge work per pass by bytes, not rows; checkpoint more often on hot bodies; move the merge off the output-gate path. | -| G26 | `checks.ts` treats an empty-string hash header as a claim, producing a false inv7 "VIOLATED" on X3-relay (5 MB body, no claim above 256 KiB by design). | **open, harness only** | Text and SV are equal everywhere; not a server bug. Fix the checker to treat "" as no claim. | - -**Base findings at full n** (not relay defects, but they bear on the comparison): -- **Base semantic reset fires on a realistic 10k-edit trace** (C2-stress-base): deleted ratio 0.311 at 20,478 - structs crosses the soft threshold, epoch 2 at frame 10,226, 4409 to all 6 sockets. With base, every client gets a - mid-session rebase on a heavily edited note. Relay has no server-initiated reset. -- **Base 5 MB compact returns HTTP 500** (X3-base, 164 ms). Relay compacts 5 MB in 846 ms. -- **Base exceeds 10 ms CPU per WS message** (C2-quick p99 30.7 ms, p90 16.8 ms; C1 p99 64.3 ms). -- **B1-base lost all sockets after 150 s idle** (1008 "socket authority mismatch"). - -**Test-environment note.** Cloudflare caps an account at 500 Durable Object namespaces (error 10067). The full run hit -it, so 40 phases (C3, B2–B6, B8, SEED, all L, B7, X2) ran on reused idle `yaos-relay2-v1001-*` workers, each with a -fresh vault and redeployed vars. The server code is identical. A self-hosted single-vault deployment is far below this -cap; it matters only to harnesses that deploy one Worker per scenario. Other run trims (user-approved): MB sweep cut to -mb {0, 10, 50}; C2-stress replayed at max rate (CPU/rows only); 10 MB X3, X2 above 800/s and X1 above 2,000 not run; -C2-stress-base gql provisional (10,226 frames); K1-alarm inconclusive. - -**Not a server bug: the smoke data-loss finding (round 4).** In smoke mb0 (v0930), B missed edits. The root cause was -transport/platform plus the raw harness client, not server merge logic (section 7). The harness was fixed in -`35b2882`. The full-n runs use the fixed client and report closes per run. - -**Risks:** - -- **R1 — Lone-surrogate data loss in ywasm Doc round-trips (K3).** - - When an update splits a UTF-16 surrogate pair, a round-trip through a ywasm Doc drops the text after the lone - surrogate (`x�Zy` → `x�Z`). Stateless ops and JS yjs preserve it. - - This **affects today's authoritative path**, which applies every body update to a ywasm Doc and re-encodes step2 - and checkpoints from it. In relay mode it affects only the lazy hash (G4) and root/Canvas docs. - `syncDocumentCache` no longer applies updates (G5, round 3). - - Real editors rarely emit split pairs. An upstream y-crdt issue plus a regression fixture are needed independent - of this RFC. -- **R2 — Starvation of the exact-head reset CAS.** - - `coveredSequence === latest_sequence` fails whenever any append lands between the client's build and the install. - - On a hot note with a slow mobile upload (5 MB: up to 116 s) the reset may never succeed. - - Options: - - a short-lived append pause for the body during install: the lease blocks appends for ≤ N s, and the server - returns backpressure - - a covered-sequence rebase: the server appends the tail rows > covered onto the new epoch, if they can be - transcoded, which they cannot across lineages without a client - - retry with a smaller snapshot - - Needs a decision. -- **R3 — The 96 MiB Wasm cap means only clients can compact bloated notes.** - - The 5m fixture (8.7 MB, 246k structs) traps the server (`unreachable`) in a Doc. - - Stateless merge high-water is ≈ 4–5× merged size, so ≈ 20 MB merged is the cap even without a Doc. The 9 MiB - merge budget keeps the server below it. - - Past that, a body is served unmerged and never checkpointed until a client resets it. A vault whose devices never - run the reset policy keeps that body's tail forever. -- **R4 — Estimated SV for bodies > 256 KiB.** G8 is the correctness side. The performance side: exact SVs above - 256 KiB cost 1.2 ms (1 MB) to 14 ms (10 MB) per append (K3), which exceeds a 10 ms Worker-style budget. The - incremental SV is therefore needed, and its drift must be measured (`stateVectorDrift` in C2/stress). -- **R5 — Write amplification against the free tier (section 9).** - - Before lean rows: one append per frame at 9–11 CF rows, ≈ 55 rows per typing-second at 5 frames/s. The model put - a typical 2 h editing day at 119% of 100k rows/day. - - Full run (inferred from measured f1001 inputs, section 9): relay lean + mb 10 is 53–73% of 100k rows/day on a - typical day, against base 30–46% (≈ 1.7×). Measured rows/edit: relay 5.00 (C4 counter) to 5.92 (gql), base 3.10. - - Strict (lean off, mb 0) is 107–119%. Lean is required on Free. - - Residual risk: - - continuous typing above ≈ 0.69–0.94 h/day for the relay (base 1.10–1.67 h), the 100k crossing point - - import days: a typical day + 2,000 notes is 115–135% for the relay, 88–103% for base - - **v3 retires R5 for edit traffic.** Section 9.1: typical 3%, heavy 18%, 19 h of non-stop typing to 100k. The import-day residual is set by - lifecycle rows (`import_rows_per_note`), which v3 does not change. -- **R6 — Tail CPU measurements are lossy.** - - `wrangler tail` dropped every WS event on a busy Worker (scratch-1 C1: 0 events). It matched 15/15 on a quiet one - (scratch-2). - - C1/C2 numbers must come from gql aggregates (`Invocations`, µs) or from matched-sample tails with loss reported. - Per-event p50 from a lossy tail is biased toward quiet periods. -- **R7 — Receiving-side validation does not exist yet.** Relay mode without it drops today's server-side - frontmatter/root-shape gate. -- **R8 — Measurements taken in Node** (K3, K2) and on an enterprise account. Workers Free DO limits are the same - tables, but CPU and memory accounting on the edge may differ. -- **R9 — Relay p99 tails and alarm CPU growth (new at full n; G24, G25).** - - L4-relay p99 2,305 ms and C2-quick-relay propagation p99 9,132 ms, while p50 stays ≈ 64–74 ms and lane-B mb 10 - p99 is 316 ms. - - The checkpoint alarm's CPU scales with body history (p99 1.04 s at 50k edits) and blocks the output gate. - - This is GO condition 2: fix or explain before default-on. -- **R10 — DO requests per edit burst are higher on the relay** (C5: 1.88 vs 0.51 units per burst; catch-up 3.43 vs - 1.92). The model puts the typical day at 1.15× base, over the 1.1× reversal bar, but at 2–3% of Free. The real - base candidate path costs more per burst (L5b1: base prod 1.05 vs relay 0.10 units), which the C5 raw adapter does - not exercise. - - **v3 does not change it.** Group commit changes rows, not WS messages: the typical day is 1.15× and heavy 1.34× - (inferred). Measured with the real client, L5b1-v3 native is 0.10 units per edit vs base prod 1.05. B5 halves - the WS messages (L5b8 0.80 → 0.40 units per 8-edit burst), but it is not recommended (R13). -- **R11 — v3: a revoked device's buffered frames reach peers but never become durable (B4-v3, measured).** - - In v3, frames broadcast before a revocation and dropped at flush (G2) leave the live peers ahead of the server. - B4-v3 dropped 10 of 40 pre-revocation frames, and its convergence check failed (fresh C and GET ≠ live - clients). This is the accepted design change "revoked frame may already have reached peers" - (`relay3-group-commit.md`, invariant change 2). Under reversal condition 4 it is still a convergence failure. - - The drop also buys little. The peer's next re-sync (reconnect step2 or wake re-sync) re-submits those bytes - under the peer's own authority, and they become durable anyway. - - **Fixed locally in d3b4db8 (not re-measured on a deployment).** (a) Primary: `flushForAuthorityFence()` runs in - the same DO turn, before `revokeDevice` / `installAuthorityFence` write authority. Already-broadcast pre-fence - frames therefore commit before the fence, and any later frame fails `validateActorCached` (the authorityVersion - bump invalidates the cache) and is refused 4403 before broadcast. (b) Safety net: if a buffered frame's device is - found revoked at flush, v3 commits it (it was admitted and broadcast) and closes the socket. Counters: - `authorityFenceFlushes`, `revokedBroadcastCommits`; `authorityDrops` stays 0 on v3. v2 micro-batch still drops, - because it broadcasts only after commit. Flushing only in `closeDevice` was rejected because the fence is - vault-wide (membership/credential revisions), not per-device. Tests (`tests/server/relay3-group-commit.ts`): - revocation with 10 buffered frames, revocation racing broadcast, and wiring order flush → fence/revoke. Peers - and durable state converge, and the post-fence frame is never broadcast or appended. -- **R12 — v3: flood isolation is flaky (B7-v3 1/3 pass).** - - In a 5 MiB/s flood of 64 KiB frames, the v3 flooder could push 63 MB (a3) before our `relay rate limit` closed - it, 43 s in. f1001 v2: 2.2 MB, closed at 26.9 s. - - In a1 the flooder was not closed within 40 s, and Cloudflare shed the bystander's socket on the same DO with - `1013 Service overloaded`. a2 was inconclusive (1006 right after a redeploy). - - Suspect: every ≥ 64 KB frame triggers a synchronous byte-cap commit, and admission keeps accepting (and - broadcasting) while the commits back up. - - Cause (code reading; the timings above are measured, the cause is inferred): the bucket was checked at - step 4 of `handleSyncFrame`. Before it, each frame paid the attachment parse, `JSON.parse` of the envelope - (envelopes were never charged), decode, SHA-256 of the whole update, the authority check and a - `candidateReceipt` SQL read. It charged `update.byteLength` only (envelopes, step1, empty frames and replays - were free). Each 64 KiB frame also hit the 64 KB cap and committed synchronously. So the DO processed the - queue more slowly than it filled, and the bucket (which refills over processing time) reached overdraft only - after the backlog. After the close, queued frames still paid decode and digest (v2 had no failed-socket mark). - - **Fixed locally (4a34f56):** `VaultSocketService.message` charges the raw message size as its first step: a - WeakMap lookup by socket, the size cap (1009) and the bucket. On the first overdraft: backpressure, close 1013 - `relay rate limit` and the cumulative-ack fence. Every later message of that socket is dropped in O(1), with - no parse, decode, digest, authority check or broadcast. WebSocket messages bypass the Worker after the - upgrade, so this is the earliest point there is. - - Local (virtual time, defaults): a 5 MiB/s flood of 64 KiB frames is closed after 375 ms of input. 740 later - messages are dropped in O(1), and bystander frames on the same body all commit, ack and broadcast. A DO-level - test uses spies to assert that drops never reach attachment parse, frame handling or authority. Not deployed. - - Still open: about one burst (≈ 1.8 MB) of frames is accepted with full work; a deep inbound queue can still - make the platform shed the DO; the budget is per socket, not per device, and a reconnect gets a fresh burst. - Envelopes and replays are now charged, so a reconnect resend backlog above the burst is refused (1013) and - resent later. - - Re-run B7-v3 serially on an idle machine. -- **R13 — v3: B5 send-coalescing defeats group commit at idle = 300 ms (measured).** With B5 at 250 ms, wire frames - are ≥ 250 ms apart. The 300 ms idle timer then fires between most of them: MB type5 went from 70 to 275 commits, - 0.38 → 1.50 rows/edit (3.9×), and added 250 ms to propagation (L2 48 → 301 ms p50). Keep B5 off. If it is ever - wanted, idle must be well above the coalesce window (≥ 2×). -- **R14 — v3: receipts are 0.3–1.5 s after the edit (user-approved).** Continuous typing at 5 keys/s commits on - max-wait, so the receipt p50 is ≈ 0.9 s and p99 ≈ 1.6 s (MB type5). The UI must not treat "unconfirmed for < 2 s" - as pending-sync noise. The client resend timer (5 s) and the HTTP fallback (15 s) sit well above the 1.5 s max. - -**Open questions:** - -- **Q1** (Answered in round 4 by lean rows: catalog events are coalesced per body per alarm pass.) Is a catalog delta - feed lag of ≤ 2 s acceptable to every reader, or do some need per-append events? - Currentness then needs `latest_sequence` from the head, which it already has. -- **Q2** Should step1 always answer by sequence (unmerged parts) rather than by SV diff? That trades bytes for E2EE - readiness and zero merges. -- **Q3** A poison-pill policy for bodies whose bytes make merges throw: quarantine, and surface to clients for a reset? -- **Q4** Lease TTL vs upload size: should the TTL scale with the declared snapshot size, or should an upload in - progress extend the lease? -- **Q5** Keep attribution per frame or per append? Per frame is N rows per batch. Lean keeps frame 0 inline and - prunes it at the feed floor. Is losing long-term per-edit attribution acceptable? -- **Q7** Should turning lean off be supported (it needs a forced coalescing pass first)? -- **Q6** Rollback relay → base on the same storage: allowed or forbidden? - -## 13. Reversal conditions - -Abandon the change or roll back to base if **any** of the following holds at full n on the Workers Free-representative -config: - -1. **Latency.** Relay L2 p50 is not at least 2× better than base, or relay L2 p99 is worse than base p99. -2. **Rows written.** With measured C4 on the lean + mb 10 config, the costmodel typical day (2 h, 3 devices, - 5 frames/s) is > 100% of 100k rows/day for the relay **and** > 1.25× base. -3. **DO requests.** Idle or typical-day DO requests for the relay are > 1.1× base (C5). -4. **Correctness.** Any convergence failure (section 8.7), any violation of invariants 1–5, 3a, 7, 8 or 11 in tests - or deployed runs, or any frame-accounting identity mismatch. Examples: an append after the revocation fence - sequence (B4), an old-epoch append (B5), more than one append per resent candidate (B6), or an unaccounted frame. A - loss traced to the platform or transport with the resilient client in place counts too. -5. **CPU.** Relay C1 heavy-note CPU per keystroke p50 is not below base's, or relay p99 per append is > 10 ms on a - ≤ 256 KiB body. -6. **Memory and scale.** Relay X1 is below 250 concurrent body sockets per vault, or relay C3 Wasm high-water at - 32 × 512 KiB exceeds base. -7. **Reset.** Mobile-class inferred reset build p50 for 1 MB is > 2 s, or B5 at full n shows a lost edit, or reset - starvation (R2) persists: fewer than 90% of reset attempts on a note edited at 1 frame/s succeed within 3 tries. -8. **Stored-state growth.** After the G1 fix, `vault_journal` rows per relay body are not bounded (more than - checkpoint entries + feed retention) after the stress trace. -9. **Post-launch regression.** A support-visible rise in conflict copies, attributable to rebase, of more than 2× base - in the first release. - -**Status at full n (f1001):** - -| # | Status | Evidence | -|---|---|---| -| 1 | not triggered | L2 p50 308 → 66.0 ms (4.67×), p99 608 → 314 ms (measured, n = 290) | -| 2 | not triggered | relay 53–73% of 100k rows/day (inferred, measured inputs); > 100% only for strict (107–119%) and import days | -| 3 | **exceeded on the letter; needs sign-off** | model typical day 2,648 vs 2,300 DO req/day = 1.15× base, both 2–3% of Free; idle equal (1,056). Measured C5 per burst 1.88 vs 0.51 units (raw adapter); L5b1 with the real candidate path 0.10 vs 1.05 units (R10) | -| 4 | not triggered | relay convergence 9/9, strict 3/3; invariants 1–8, 3a, 10, 11 hold; accounting 16/16. The base C2-stress FAIL is base's own semantic reset | -| 5 | not triggered | C1 WS CPU p50 0.22 vs 1.36 ms; relay WS p99 4.2–4.6 ms ≤ 10 ms. The checkpoint alarm (p99 up to 1.04 s) is not per append and is tracked as R9 | -| 6 | not triggered | X1 2,000/2,000 ≥ 250; C3 4.06 MB < 37.22 MB | -| 7 | partly open | 1 MB mobile build 103–155 ms inferred (< 2 s); B5 20/20, no lost edit. Starvation (R2) at 1 frame/s: not measured | -| 8 | not triggered (partial evidence) | K1-compact → 0 log rows; unit test 5,000 appends → 1,001 rows. Per-body journal row counts after the stress trace: not reported in the full run | -| 9 | n/a | post-launch | - -**Status for relay v3 (r3-1002/b/c, measured unless noted):** - -| # | Status | Evidence | -|---|---|---| -| 1 | not triggered | L2 p50 299 → 47.7 ms (6.3×), p99 696 → 76.3 ms ([conc], n = 290). The receipt is slower by design (p50 350 ms, R14) | -| 2 | not triggered, and the margin is now large | v3 is 3% typical and 18% heavy (inferred); 0.29 rows/edit vs v2 5.63 (a 5 keys/s best case; slower typing cost up to 3 rows/key before the 4a34f56 commit-rate cap, section 9.1). Condition 1 of section 1 (lean mandatory) is retired | -| 3 | **unchanged: exceeded on the letter; needs sign-off** | group commit does not change WS messages. Model 1.15× base typical, 1.34× heavy (inferred), ≤ 11% of Free. R10 is unchanged | -| 4 | **triggered on the letter at measurement (B4 convergence); fixed locally, re-measure pending** | invariant 4 holds (0 appends after the fence). At measurement, a revoked device's pre-fence frames were broadcast and then dropped at flush, so 10 frames diverged (R11). d3b4db8 flushes before the fence and commits rather than drops at flush; local tests converge. Not re-run deployed. Convergence suite 9/9, CRASH 10/10 with 0 lost, FENCE 6/6 refused rounds safe. B7 1/3 (R12) | -| 5 | not measured for v3 | C1 not re-run. The v3 flush is one batch per 300 ms–1.5 s, so per-append CPU is no worse than v2 (inferred) | -| 6 | not re-run | X1 not run on v3. CW 121/121 pass | -| 7 | B5 20/20 | no lost edit across epoch resets under v3. R2 not measured | -| 8 | not re-run | the commit-row shape is the same as v2 | -| 9 | n/a | post-launch | - -## 14. Appendix - -### 14.1 Harness usage - -See `scripts/relay2/README.md`. In short: - -``` -zsh scripts/relay2/deploy.sh yaos-relay2- --relay on|off [--require-clean | --src ] -node tests/run-typescript.mjs --test-aliases scripts/relay2/context.ts --host -node tests/run-typescript.mjs --test-aliases scripts/relay2/bench.ts --host --out \ - --adapter base|relay|relay-nosv|relay-nocand [--tail] -python3 scripts/relay2/analyze.py # summaries -node tests/run-typescript.mjs --test-aliases scripts/relay2/gql.ts … # CPU from analytics (preferred, R6) -python3 scripts/relay2/costmodel.py [--set k=v] [--json measured.json] # section 9 -``` - -Other entry points: -- Reset and K2: `scripts/relay2/reset/`. -- Byte ops and K3: `scripts/relay2/k3/`. -- Core smoke: `scripts/relay2-core-smoke.ts `. - -Local tests (round 4 unless noted): - -| Suite | Result | -|---|---| -| `tests/server/relay2-server-core.ts` | 31/31 in default and lean modes (round 3: 27/27) | -| `tests/server/relay2-byteops.ts` | 9/9 | -| `tests/client/relay2-reset-builder.ts` | 45/45 | -| `tests/client/relay2-reset-policy.ts` | 31/31 | -| `tests/client/relay2-reset-race.ts` | 127/127 | -| flag-off regressions | 3,635 pass, 0 new failures (round 2: 3,622/0; round 1: 199/199 steps) (`results/relay2/flag-tests.md`) | -| flag on (default and lean) | only the expected capabilities key-set failures. Lean matches after the lazy-ALTER fix. Headless 92/92 in both modes (round 3). Conformance 13/13 flag off (round 3). | -| base suites (base SHA) | 179/179 suites, 3,135 assertions; node-runtime 32/32; conformance 26/26 (1 declared GAP); headless 92/92 (1 of 4 flaky) (`results/relay2/base-tests.md`) | - -### 14.2 Raw data index - -| Data | Location | -|---|---| -| Full-n run JSONs (per scenario id) | `experiments/logs/relay2/runall-f1001/raw/.json` (tag f1001, 2026-10-01); `tables.md`, `progress.jsonl`, `runall.log`, `gqlfill-final.log` alongside. Manifest copy: `experiments/results/relay2/raw/runall-manifest.jsonl` | -| Small-n / smoke run JSONs | `experiments/logs/relay2/runs/`, `experiments/logs/relay2/runall-w0930-small/raw/` (not in git; contexts hold tokens) | -| Deploy records | `experiments/logs/relay2/deploy-.json` | -| Results tables | `experiments/results/relay2/RESULTS.md`; report `REPORT.md` | -| K2 | `experiments/results/relay2/K2.md`, `K2-local-2026-09-29T19-58-32-456Z.json` | -| K3 | `experiments/results/relay2/K3.md`, `K3-byteops-all.json`, `K3-byteops-svdiff-all.json`, `K3-bundle-size.json` | -| B5 | `experiments/results/relay2/B5-2026-09-29T20-13-40-001Z.json`, `B5-2026-09-29T20-16-32-720Z.json` | -| Live reset (round 2/3) | `experiments/results/relay2/core2-live-reset-5mb.json`, `core3-live-reset-5mb.json` | -| Flag / base tests | `experiments/results/relay2/flag-tests.md`, `base-tests.md`, `base-tests.json`, `core{,2,3,4}-flag-*` logs | -| Prior art | `experiments/results/A1.md`, `A2-A4.md`, `A3.md`, `A5.md` | -| Convergence | `experiments/logs/relay2/runall-f1001/raw/convergence-suite.json` (relay 9/9), `convergence-suite-strict.json` (3/3), `convergence-suite-base.json` (6 pass, 1 fail, 1 n/a, 1 skipped) | -| Final deploy version IDs (base / relay / strict) | Tag f1001. Lane B: fresh `yaos-relay2-f1001--` workers at spike `d929410`. Reused phases: `yaos-relay2-v1001-*` workers redeployed at `45f3705` with per-phase vars (the worker name is historical; the variant is recorded in each JSON's `vars` and `protocolAdapter`). Every raw JSON records `workerName`, `deploymentVersionId`, `baseSha` and `deployedSpikeSha`. Selected below. | - -Selected deploy records (from raw JSON metadata; base SHA `5dd32f3` throughout): - -| Scenario | Worker | Deployment version id | Spike SHA | Started (UTC) | -|---|---|---|---|---| -| L2-base | `yaos-relay2-v1001-c1-strict` | `c3bcd97d-f79c-4aa1-8f47-be9d03d4545f` | `45f3705` | 2026-10-01 15:13 | -| L2-relay | `yaos-relay2-v1001-c2-quick-relay` | `ebb06898-a491-4cc0-9a59-15f816046962` | `45f3705` | 2026-10-01 15:15 | -| L2-strict | `yaos-relay2-v1001-c2-quick-base` | `b9e93e19-f7cf-4565-8d2b-3eb78f55ef4a` | `45f3705` | 2026-10-01 15:18 | -| L4-base | `yaos-relay2-v1001-l1-relay` | `007698a0-8b09-461e-9dde-e9813fa58a0f` | `45f3705` | 2026-10-01 15:29 | -| L4-relay | `yaos-relay2-v1001-c3-base` | `aef12799-e63a-4e38-8078-76dc16b40614` | `45f3705` | 2026-10-01 15:29 | -| L4-strict | `yaos-relay2-v1001-c2-stress-relay` | `de5bb064-881e-4f43-97c1-569575d74c33` | `45f3705` | 2026-10-01 15:24 | -| C1-base | `yaos-relay2-f1001-c1-base` | `4b67ae31-f85b-46bb-a68a-7bb87bfda1a6` | `d929410` | 2026-10-01 13:48 | -| C1-relay | `yaos-relay2-f1001-c1-relay` | `be47a71b-43c8-434a-9851-76c978347368` | `d929410` | 2026-10-01 13:48 | -| C1-strict | `yaos-relay2-f1001-c1-strict-a2` | `0ef690be-fde1-4c95-b6c4-95c4e7c22476` | `d929410` | 2026-10-01 13:53 | -| C2-quick-base | `yaos-relay2-f1001-c2-quick-base` | `52b8570b-6ef9-428b-a5da-85e38555d16e` | `d929410` | 2026-10-01 13:46 | -| C2-quick-relay | `yaos-relay2-f1001-c2-quick-relay` | `7259077d-3faa-44cd-92b5-b96110e14a4a` | `d929410` | 2026-10-01 13:46 | -| C2-quick-strict | `yaos-relay2-f1001-c2-quick-strict` | `15175c87-d554-4cf0-a412-bf30f871b0e5` | `d929410` | 2026-10-01 13:47 | -| C2-stress-relay | `yaos-relay2-f1001-c2-stress-relay` | `07dbcf83-05a3-4e2a-9b88-3f77d5d59a35` | `d929410` | 2026-10-01 13:38 | -| X1-relay | `yaos-relay2-f1001-x1-relay` | `5f82c76b-4a8a-4112-8504-f9fd086703fb` | `d929410` | 2026-10-01 13:37 | -| B5-relay | `yaos-relay2-v1001-b7-relay` | `e2fae91c-386d-4680-97f5-670c437c4108` | `45f3705` | 2026-10-01 15:08 | -| C6-relay | `yaos-relay2-v1001-c1-base` | `ee378cd6-1574-424f-a7d2-d04f6fdd1f23` | `45f3705` (harness `35e8b06`) | 2026-10-01 15:50 | diff --git a/docs/server-rewrite/DECISIONS.md b/docs/server-rewrite/DECISIONS.md new file mode 100644 index 00000000..d330d82b --- /dev/null +++ b/docs/server-rewrite/DECISIONS.md @@ -0,0 +1,772 @@ +# YAOS server rewrite: decisions + +Status: approved 2026-10-06; implementation under way (section 9). Branch `server-remake`. Wire baseline: +`docs/client-remake/relay-wire.md` ("the contract"). IDs are stable. The review flags F1–F18 are folded into the +decisions; section 8 logs where each one landed. + +Budget profile: Cloudflare Workers Free. 100k rows written/day, 5M rows read/day, 100k DO requests/day, 10 ms +CPU, 5 GB DO storage, 2 MB max row. PITR keeps 30 days and does not exist in local dev. + +--- + +## 1. Scope and non-goals (D1) + +- **In scope.** One person, many devices: one operator per server, any number of vaults. +- **E2EE.** Single-user E2EE (client-side CryptoPort suite 1) is in scope, so the server stays fully opaque. It + never interprets payloads, checkpoints, stream names or blob contents. +- **Cloudflare only, fresh deploy.** The core keeps small injected ports (storage, sockets, clock) so tests run it + on Node. No migration, no dual writes; clients re-seed from disk. +- **Removed (not deferred).** Members, invitations, principals, ownership, governance, device-links, the read-only + role; shared E2EE vaults; server-side CRDT (semantic sockets, candidates, bootstrap, catch-up, compaction, + snapshots, recovery jobs); settings sync, relay v2 bodies, bulk create, update-metadata; `packages/server-node`, + the Docker image and `packages/cli` (already deleted on this branch, §2.3). +- **Size.** Keep about 2.2k lines, write about 3k. Delete about 42.8k in `server/src`, 4.1k in + `packages/server-node` and 5.3k in `packages/cli`. + +--- + +## 2. Architecture + +### 2.1 Responsibilities + +| Component | Owns | Never does | +|---|---|---| +| Worker (stateless) | Route table; method/path match; size caps; format checks (vaultId `^[A-Za-z0-9_-]{22}$`, pairing code, blob address, `streamsVersion=1`); body streaming to the DO; unread-body drain in `finally`; R2 I/O for blobs; `claimed=true` cached in isolate memory forever once seen | Crypto; config-DO reads on device routes; storage writes | +| Vault DO (`idFromName(vaultId)`) | `vault_meta` (vaultId, generation = epoch, ticket key); devices; pairing codes; bearer auth; ticket issue and verify; streams store and relay; dedupe index; per-device rate and socket caps; revoke; reset; restore steps; daily-limit latch | Calls to the config DO | +| Config DO (singleton `idFromName("config")`) | Operator credential hash; operator sessions; vault registry (id, name, createdAt); restore journal, steps and alarm (D8b); claim; in-memory login-failure limiter | Being on the hot path. Only `/claim`, `/operator/*` and pre-claim `/api/capabilities` reach it | + +Gone: the config→vault authority mirror, the authorization fence, `202 authorization_fence_pending`, the 5 s +actor-authority TTL cache, vault lifecycle states, and the `YAOS_STREAMS` flag (streams are always on). + +### 2.2 Route table + +Auth: `-` none, `C` operator session cookie, `B` device bearer, `T` streams ticket. Everything not listed → `404 +{"error":"not_found"}` with zero DO calls. + +| Route | Auth | Worker work | DO(s) | +|---|---|---|---| +| `GET /`, `GET /mobile-setup` | - | static HTML (D5) | none | +| `OPTIONS` on `/api/*`, `/enroll`, `/vault/*` | - | static CORS preflight, 204 | none | +| `GET /api/capabilities` | - | answers from the isolate cache once `claimed` | config, only until `claimed` is seen | +| `POST /claim` | - | key ≥ 32 chars; JSON ≤ 64 KiB; mints vaultId | vault (init), config (claim), vault (owner code); order in D5 | +| `POST /operator/login`, `/operator/logout` | -, C | | config | +| `GET /operator/state` | C | | config (also lists pending restore journal rows) | +| `POST /operator/vaults` | C | mints vaultId; no `/provision` (D5) | vault (init), then config (register) | +| `POST /operator/vaults/:id/owner-code` | C | | config (session, registry), vault | +| `DELETE /operator/vaults/:id` | C | `{"confirmVaultId"}`; R2 prefix purge; order in D5 | config, vault | +| `GET /operator/vaults/:id/devices` | C | | config, vault | +| `DELETE /operator/vaults/:id/devices/:deviceId` | C | | config, vault | +| `POST /operator/vaults/:id/reset-streams` | C | | config, vault | +| `POST /operator/vaults/:id/restore` | C | | config (session, journal, steps), vault | +| `POST /enroll` | code | code regex; vaultId taken from the code | vault | +| `POST /vault/:id/auth/pairing-code` | B | | vault | +| `POST /vault/:id/auth/ticket` | B | | vault | +| `GET /vault/:id/ws/streams` | T | `streamsVersion` check (1008 `update_required`) | vault | +| `GET /vault/:id/streams/feed`, `/streams/read` | B | | vault | +| `PUT /vault/:id/streams/checkpoint` | B | Content-Length ≤ 4 MiB else 413; streams body | vault | +| `PUT`/`GET /vault/:id/blobs/:addr`, `POST /vault/:id/blobs/exists` | B | address regex; ≤ 10 MiB; R2 I/O | vault (bearer check only) | +| `POST /vault/:id/debug/simulate-daily-limit` | B | only when `YAOS_DEBUG_ROUTES=1`, else 404 | vault | + +Device routes on an unknown vault answer as for a bad credential: `401 unauthorized` (at upgrade: `unauthorized` +frame + 1008). There is no existence oracle, and it matches revoke and §10's "unauthorized after a delete". +*Test:* T-UNKNOWN-VAULT-401. + +### 2.3 Deleted (by subsystem, `wc -l` before deletion) + +| Subsystem | Files | Lines | +|---|---|---| +| Semantic document plane | vaultDocumentStore, vaultDocumentCache, vaultSocketService, vaultSemanticService, vaultCandidateService, semanticCompaction*, vaultBootstrapStore, vaultBulkCreateService, vaultLifecycleService, vaultCatalogStore, vaultStore, vaultObjectStore, bootstrap, contracts | 12,415 | +| Recovery | recovery*.ts, vaultRecoveryService | 9,955 | +| Relay v2 spike | relayBodies, relayBodyStore, relayRoutes, relayTail, relayFlag | 3,316 | +| Collaboration/governance | collaboration*, controlPlaneSql, vaultAuthority | 2,744 | +| Legacy config + identity | config, identity (rewritten as the config DO) | 2,449 | +| DO host | server.ts (rewritten as the vault DO) | 2,241 | +| Worker + routes | index.ts, routes/* (rewritten) | 2,636 | +| Shared CRDT/canvas codecs | shared/* except socketLiveness, socketCloseCodes | 2,086 | +| CRDT glue | crdt/* | 1,512 | +| Setup page | setupPage.ts (replaced by the console) | 1,445 | +| Settings sync | settingsSyncStore | 880 | +| Bench/debug | storageBenchmarkWorker, sqlRowCounter, testOnlyTimers | 697 | +| Node host | packages/server-node; Dockerfile, compose.yaml, .dockerignore, tests/docker/smoke.mjs, Docker CI and release jobs | 4,088 | +| Legacy CLI | packages/cli (the schema-6 daemon) | 5,282 | + +The Node host and the legacy CLI are already deleted on this branch. The trimmed SQLite adapter that +`streams-relay.ts` and `cfRowModel.ts` need lives in `tests/server/helpers/`. The `server/src` rows go in P1. +*Why:* code excluded from the build rots; git log keeps it if it is ever wanted back. + +Kept (about 2.2k): streams/{protocol,relay,store} 1,387; dailyLimit 263; socketLiveness 260; readBoundedBytes +138; base64url 76; setupQr 29; hex 24; version 14; socketCloseCodes 2. `vaultId.ts` is rewritten to the strict +22-char check. + +New code (about 3k): Worker router 350, vault DO host 700, config DO 300, strict codec 100, relay deltas +(H2/H3/H5/H6/H7/H8) 400, blobs 120, console 450, WB tests 600. The BB conformance suite is extra. + +--- + +## 3. Decisions D2–D9 + +**D2 Topology.** As in §2.1. `/api/capabilities` needs only `claimed`, and the Worker caches `true` forever. +*Why:* the config DO was a serial hop on every device request (§15: ticket 107 ms p50 deployed) and the source of +the fence. *Tests:* T-ENROLL-200 (BB), T-HOTPATH (WB: feed, read, checkpoint, ticket, socket, enroll and +pairing-code make zero config-DO calls, checked with a counting stub). + +**D3 Pairing codes and enroll.** +- Format `.`: base64url(16 B) = 22 chars, `.`, base64url(24 B) = 32 chars; 55 in total. +- The Worker trims and checks `^[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{32}$`. Malformed → `400 invalid_code`, no DO call. +- The vault DO looks up SHA-256(secret) by PK. An unknown vault and an unknown secret both → `404 invalid_code`; + an unknown vault costs zero writes and no DDL. Expired → `410 expired_code`; used → `409 used_code` (today's + statuses). +- **Replay (mandatory).** The code row keeps `used_request_id` and `used_device_id`. The same + `enrollmentRequestId` + deviceId + token hash on the same code, while that device row still exists → the same + 200 body, 0 rows. `deviceName` is not part of the key (a rename-only retry replays the stored name). The same id + with another deviceId or token → `409 enrollment_request_conflict`. The device row is gone (revoked) → `409 + used_code`: a replay never resurrects a device. *Why:* if the 200 drops on the network, the client retries; + without replay it gets `used_code` and the pairing is lost. +- **200 body.** Exactly `host, deviceToken, vaultId, deviceId, deviceName, vaultGeneration`: the fields + `readEnrollment` reads (`src/host/ui/pairing.ts`). The D6 constants belong to VAULT_READY only. +- TTL 15 min. Purposes: `owner-bootstrap`, `owner-recovery`, `device`. Limiter: in memory, 20 failures/min per + vault DO → `429 too_many_attempts`. +- The server never sees key material. A future client-only key part must be stripped before `/enroll`. +- *Why:* routing by code removes the config lookup, and 192 bits make brute force moot. *Tests:* T-PAIR-FORMAT, + T-PAIR-MALFORMED, T-PAIR-UNKNOWN-VAULT, T-PAIR-USED, T-ENROLL-REPLAY, T-ENROLL-CONFLICT, T-ENROLL-BODY (BB), + T-PAIR-NOWRITE (WB). + +**D4 Tickets.** +- Key: 32 random bytes per vault in `vault_meta`, made at init, never exported or rotated. The vault DO issues + tickets and verifies them at upgrade, with the key and device map in memory. +- Payload as today (vaultId, deviceId, D6 constants, aud, purpose, documentId, iat, exp, nonce), signed with the + vault key. Wire unchanged: aud `yaos-vault-ws`, purpose `streams`, TTL 5 min (`YAOS_TICKET_TTL_MS`). +- Issuing costs 0 rows (the lastSeen "touch device" write is dropped). Upgrade rejections keep the §3.1 shape + (accept, `error` frame, close 1008); the DO now produces them. +- The §3.1 `unclaimed` frame is never sent: it would need a config read at upgrade. An unclaimed server has no + vaults, so the DO answers `unauthorized`. +- *Why:* a key per vault makes cross-vault replay impossible by construction. *Tests:* T-TICKET-CROSS-VAULT, + T-TICKET-BAD (BB). + +**D5 Surviving routes.** Exactly the §2.2 table; everything else → 404. +- Static Worker responses, no DO call: the console (`GET /`), `GET /mobile-setup` (the target of `mobileSetupUrl` + and the console's setup QR) and CORS preflight. +- The console is one server-rendered page with inline JS and no external assets. It covers claim, login, the + vault list, create vault, owner code + QR (drawn in the page, O12), devices + revoke, reset streams behind a typed vaultId + confirmation, and the D8b "Restore incomplete" banner. +- Operator JSON routes require `Content-Type: application/json` and a same-origin `Origin`. The cookie is + HttpOnly, Secure, SameSite=Strict, 7 days. +- **Claim:** vault init → config claim (operator, vault and session in one write) → owner code. A failure after + the claim → 503; the user re-probes, logs in and uses owner-code. +- **Create vault:** no `/provision` (→ 404). Idempotent vault init first, registry row last, so a listed vault is + always initialized. A failed registration leaves an unreachable 1-row orphan DO (accepted). +- **Delete vault:** `{"confirmVaultId"}`, missing or mismatched → `400 confirmation_mismatch`. Sockets 1001 + + `deleteAll()`, then the R2 prefix purge (1000 per batch), then the registry row; an incomplete purge → `503 + purge_incomplete`, retryable. Afterwards every device route → 401. +- *Why:* the rest serves only cut subsystems. *Tests:* T-LEGACY-404 (BB: members, invitations, governance, + device-links, settings-sync, catch-up, bootstrap/start, legacy `DELETE /operator/devices/:id`), T-DEVICES-LIST (BB: + deviceId, deviceName, no token material), T-CONSOLE-ROUTES, T-PROVISION-404, T-VAULT-DELETE-CONFIRM (BB). + +**D6 VAULT_READY constants.** +- `role:"owner"`, `canWrite:true`, `membershipRevision:1`, `deviceCredentialRevision:1`, `policyVersion:1`. +- `principalId` = `owner:`, in VAULT_READY only: the enroll body is D3's six fields (`readEnrollment`, + `src/host/ui/pairing.ts:363-388`, never reads it). +- `capabilityDigest` = today's `capabilityDigestForRole("owner")` value, frozen as a literal. +- Every §3.2 field stays; the `write_forbidden` path is removed. *Why:* the shape stays for the shipped client; + the authority model is gone. *Test:* T-READY-SHAPE. + +**D7 Device revoke.** `DELETE /operator/vaults/:id/devices/:deviceId`: operator-only, idempotent, `200 {ok, +deviceId, revoked}`. The gate shuts with the transaction, not with the close handshake. +- **Revoke turn** (one synchronous turn): DELETE the device row and its device-map entry; drop its frames from the + pending group-commit buffer (they never commit, no receipt); remove its sockets from the fanout set; send each + `__YPS:{"type":"error","code":"authority_superseded"}` and close 4403. +- **Inbound gate.** The socket attachment carries deviceId. Every `webSocketMessage` first checks the device map + (rebuilt from SQLite at wake); a miss drops the frame, sends `authority_superseded` and closes 4403. +- **Wake rebuild (added, O3).** H5's rebuild skips, and closes 4403, sockets whose device is not in the map, so a + revoked socket whose close never completed can't rejoin fanout. +- **Close.** The error and 4403 close frames leave in the revoke turn (legacy: +207 / +211 ms after the request). + The TCP end follows about 10 s after the DO's last event; that is platform teardown, measured but not judged. + The new deploy uses a compat date ≥ 2026-04-07 (`web_socket_auto_reply_to_close` on); whether that shortens the + teardown is unmeasured, and the gate does not depend on it. +- The bearer then gets 401 on every device route, ticket included; the upgrade checks the device map, so an + unexpired ticket stops working. +- *Why:* one DO, so no TTL cache and no fence. *Tests:* T-REVOKE-GATE, T-REVOKE-SILENCE, T-REVOKE-BUFFER, + T-REVOKE-4403 (error and close frame each < 1 s; TCP end recorded), T-REVOKE-401. +- *Measured on legacy* (`conformance-scratch3-revoke-20261006T051325Z.json`): 4 cycles, ~5,300 frames sent after + the revoke response, 0 committed, 0 echoed; the committed frames are always an unbroken prefix of those sent + before the fence ran. Legacy flushes the 23 buffered frames in the revoke turn (BUFFER fails, as D7 intends to + change). One cycle showed a 484 ms gap between the cutoff and the error frame, on the safe side (frames in it + were rejected), cause not found. + +**D8 Epoch.** vaultEpoch == vaultGeneration: random base64url(16), minted at vault init. Device credentials are +bound to the vaultId only. *Why:* an epoch means only "the stream space was replaced"; credentials survive it. + +**D8a reset-streams.** Request `{"confirmVaultId"}` (missing or mismatched → `400 confirmation_mismatch`) → `200 +{vaultEpoch}`. +- One `transactionSync` deletes every row of the 3 stream tables and writes a new generation. Then the pending + buffer, head cache and dedupe index are discarded and streams sockets close 1001. Devices stay enrolled. +- Cost: one row per stream row (§6.2). A daily-limit hit rolls the transaction back → `503 cf_daily_limit`. P3 + measures DROP+CREATE billing on workerd and uses it if it is cheaper. + - *Measured (P3a, local workerd, `cursor.rowsWritten`):* DROP+CREATE bills a constant 7 rows (0 per DROP, 2 per + CREATE TABLE, 1 per CREATE INDEX). Not adopted: DROP fails with SQLITE_LOCKED while any read cursor on the + table is open, and an early-exited `for…of` keeps one open until GC. Reset stays DELETE at H+S+C+1. +- *Test:* T-RESET. + +**D8b restore (approved 2026-10-06; operator-only, cold path).** `POST /operator/vaults/:id/restore {"at": ISO8601}` → `200 +{vaultEpoch}`. A restore rewinds content, never authority. +- **Why not a temp table:** PITR "appl[ies] to the entire SQLite database contents, including … key-value data" + and returns storage "to exactly match what the storage contained at the given bookmark". A backup table written + before the rewind is erased by it. +- **Why a journal outside the vault:** finish must always run. Until it does, the rewound vault serves the old + device table and the old epoch, and a client still on that epoch skips the reused seqs (§11 step 5, "skipping + seqs ≤ the stream cursor"): silent divergence. Only finish's new epoch heals it (§11 step 3 drops all cursors). + The vault cannot detect its own rewind (the restored state is a genuine past state), so the intent lives in the + config DO, with a config-DO alarm that resumes it. +- **Crosses the rewind:** the device rows only. Not pairing codes (all deleted), the epoch (finish mints a new + random one, so no collision with a post-T epoch), the ticket key or the daily latch. + +The config DO runs the steps (the Worker forwards the request after the session check): + +0. INSERT the journal row (§6.3) and set the alarm to now + 30 s. A PK conflict means a restore is pending: + resume it, ignore this request's `at`, respond with `resumed:true` and the journaled `at`. An `at` that is + unparseable, in the future or over 30 days old → `400 invalid_restore_point` before any effect; no PITR (local + dev) → `501 restore_unsupported`. +1. Vault `prepareRestore(restoreId, at)`: an in-memory `restoring` flag with a 60 s TTL (device routes and enroll + → `503 restore_in_progress` + `Retry-After`; upgrades refused); streams sockets close 1013 + `restore_in_progress`; the buffer is dropped; `vault_meta.pending_restore_id = restoreId`; return `{bookmark: + getBookmarkForTime(at), devices}`. The config DO stores both in the journal row. +2. Vault `rewind(restoreId, bookmark)`: `last_restore_id == restoreId` → already finished, go to 4. Otherwise + `onNextSessionRestoreBookmark(bookmark)` and `ctx.abort()` (the RPC throws by design). +3. Vault `finishRestore(restoreId, devices)`: + - `last_restore_id == restoreId` → no-op, current epoch. + - `pending_restore_id` still set → the rewind did not happen (a real rewind erases the marker, written after + T) → back to 2. + - any write since this runtime booted (commit, checkpoint, enroll, code) → back to 2, so content is exactly T. + - Else one `transactionSync`: replace the device table, delete all codes, mint a new epoch, set + `last_restore_id` and `last_restore_at`. Then every streams socket closes 1001. +4. Delete the journal row; `200 {vaultEpoch}` (synchronous: the operator waits a few seconds). A failure before + this → `503 restore_incomplete`, and the alarm finishes it. + +- **Resume** (a second press, or the alarm with platform retries): if the vault still has `pending_restore_id == + restoreId` (not rewound), rerun 1 to refresh the snapshot; then always 2 → 3 → 4. Step 2 re-arms the same + bookmark, which erases anything written while the vault sat rewound. +- **Authority freeze.** While a journal row exists, the config DO refuses that vault's revoke, owner-code and + reset-streams with `409 restore_in_progress` (operator routes already pass the config DO, so it costs nothing). + Vault delete wins: it deletes the journal row first. An enroll that lands in the rewound vault is dropped by the + re-rewind (fail-closed: re-pair). +- **Invariants.** Cold path only: the operator route is the sole entry, and the journal, steps and alarm never touch + a device path. The vault DO never calls the config DO; no device route gains a hop. Rows: config 3 (insert, + update, delete); vault 1 (the marker, which the rewind erases) + finish. +- **Residual window.** No crash: one internal RPC between abort and finish; upgrades re-check the device table, + and a write in the window sends finish back to 2. Crash: at most the alarm delay (≈ 30 s plus retries) of + rewound content under the old epoch, healed by finish. `GET /operator/state` lists journal rows, so the console + shows "Restore in progress". +- *Tests:* T-RESTORE-RESUME-WB (fake PITR port; crash between every pair of RPCs; final devices == pre-restore + devices, no codes, new epoch, content == T, no journal row), T-RESTORE-MANUAL (deployed, by hand). + +**D8c epoch param.** Feed, read and checkpoint take an optional `epoch=`. +- The check runs right after bearer auth, before any other validation or effect. Absent → no check. +- Mismatch (empty included) → `409 {"error":"vault_generation_mismatch","vaultEpoch":""}`. +- *Tests:* T-EPOCH-MISMATCH, T-EPOCH-MATCH, T-EPOCH-ABSENT. + +**D9 Blobs (R2 only).** +- Address `^[0-9a-f]{64}$`: format check only, no SHA-256 check. PUT overwrites. Body ≤ 10 MiB + (`maxBlobUploadBytes`), else 413. `exists` takes ≤ 50 addresses. +- No `YAOS_BUCKET` → `503 attachments_unavailable` and `capabilities.attachments=false`. +- **R2 key `v//
` (mandatory)**, with no generation. Vault delete purges the prefix. *Why:* the + content address is the identity, so identical bytes are the same blob across resets, restores and epoch bumps. +- *Why opaque:* E2EE addresses are HMAC(vaultKey, hash), which the server cannot check; the client verifies on + download. *Tests:* T-BLOB-OPAQUE, T-BLOB-KEY-RESET (both SKIP when `attachments=false`), T-BLOB-UNAVAILABLE. + +--- + +## 4. Hardening H1–H8 + +**H1 Strict codec.** It replaces lib0 `readVarString`/`readVarUint` on the APPEND path. lib0 turns invalid UTF-8 +into U+FFFD and accepts non-minimal varuints. +- Strings: `TextDecoder("utf-8", {fatal: true, ignoreBOM: true})` (`ignoreBOM` keeps a leading U+FEFF). Varuints: + minimal (no trailing `0x00` group) and ≤ 2^53. +- Every length must fit the remaining bytes, and the message must be fully consumed. +- Any violation → close 1008 `malformed_frame`. +- *Why:* two byte strings must never alias one stream name or frame id. *Tests:* T-CODEC-UTF8, T-CODEC-SURROGATE + (`ED A0 80`), T-CODEC-OVERLONG (`C0 AF`), T-CODEC-NONMINIMAL (`0x82 0x00`), T-CODEC-TRAILING, T-CODEC-VALID + (`"b:é✓😀"`). + +**H2 Dedupe window.** A resend is deduped exactly when its original is among the newest W stored-row bytes of its +stream. +- W ≥ burst + maxPayload + gcMaxBytes = 2 MiB + 1 MiB + 64 KiB ≈ 3.06 MiB. **W = 4 MiB.** +- The pending buffer stays exact and vault-wide. +- *Why:* today's window collapses to the last sealed segment, so after 3×600 KiB every frame but the last is + re-appended on resend. + +Index: +- **Shape.** One lazy index per stream: `Map`, plus a ring of + (hash, rowBytes) to trim entries older than W. +- **Build.** On the first append to a stream in a runtime, one bounded scan: the head (usually cached), then + `SELECT bytes FROM stream_segment WHERE stream=? ORDER BY first_seq DESC LIMIT 64`, stopping at ≥ W. It parses + only key bytes and skips payloads by length. +- **Maintenance.** Each commit adds its rows. **Hit.** Load the row (open: 0 rows; sealed: 1) and compare bytes. + Equal → `deduped:true` with the original seq; different → `client_frame_id_conflict` with seq. A collision on + another key is not a hit. +- **GC'd originals.** An entry with `seq ≤ gcSeq` can't be byte-compared: answer `deduped:true` with the original + seq; the checkpoint covers it. +- **Rows read, worst case.** Every sealed segment is ≥ 64 KiB (an end-of-commit seal needs ≥ 64 KiB, a mid-commit + seal > 1.5 MB − 1 MiB). So one scan reads ≤ 64 segments + 1 head = **65 rows** (≤ W + 1.5 MB), once per stream + per runtime, or once per eviction under the memory cap. + The 5M budget allows ≈ 76.9k cold scans/day; a realistic load (200 wakes × 10 hot streams) is ≈ 130k rows/day (2.6%). +- **CPU.** 200k tiny rows can exceed 10 ms. Parse keys only; P3 measures the cold scan. If it is still over, + chunk the scan and gate that stream's commits on it. + - *Measured (P3a, Node):* the worst case (4 MiB of 25 B rows, 168,467 rows) parses in 15.7 ms median, so the + fallback is built. Steps are 16,384 rows (median 1.2–1.35 ms, p95 ≤ 4.4 ms), and the window read is charged + one step unit per 256 B. Local workerd agrees (≈ 15 ms full parse, ≈ 2 ms per step); production CPU is + measured in P5. +- **Memory bound.** The minimum row is 21 B (seq 1, deviceId 1+16, cfid 1+1, len 1), so one stream holds + ≤ 199,728 entries, about 12 MB at about 60 B/entry (typical 250 B rows: ≤ 17k entries, about 1 MB). DO-wide cap: + 262,144 entries (about 16 MB), by LRU eviction of whole streams; an evicted stream rescans (≤ 65 rows). +- Update §5.4 (section 5). *Tests:* T-DEDUPE-LARGE (3×600 KiB), T-DEDUPE-SMALL (400×4 KiB), T-DEDUPE-CONFLICT, + T-DEDUPE-COLD-WB. + +**H3 Commit-failure typing.** The relay classifies its own commit errors; the `decorateControl` rewrite goes. +- Daily limit → `VAULT_ERROR cf_daily_limit {cause:"durability_failed", kind, resetAt, message, stream, + clientFrameIds}`. +- Anything else → `durability_failed` plus an additive `retryAfterMs = round(min(30000, 1000·2^(n−1)) · + (0.5 + 0.5·rand))`, where n counts consecutive failed commits in the runtime and resets on success. +- The checkpoint, feed and read handlers map daily-limit errors to `503 cf_daily_limit` themselves. +- *Why:* errors are typed at the source, so there is no latch race. *Tests:* T-COMMIT-DAILY-WB, T-COMMIT-RETRY-WB (WB, + new IDs), T-DAILY (BB, via `POST /vault/:id/debug/simulate-daily-limit`, only with `YAOS_DEBUG_ROUTES=1`). + +**H4 Authority before write_forbidden.** Resolved by design: no roles, `canWrite` always true, and the D7 gate +shuts in the revoke turn. *Tests:* covered by T-REVOKE-GATE and T-READY-SHAPE. + +**H5 Socket cache.** A `Map` with per-device lists, each attachment parsed once per socket +per runtime; maintained on accept, close and error, rebuilt from `ctx.getWebSockets()` after a wake (skipping +revoked devices, D7). *Why:* today `streamSockets()` walks every socket on every call. *Test:* T-SOCKET-CACHE-WB +(WB, new ID). + +**H6 Per-device limits.** +- One in-memory token bucket per deviceId (256 KiB/s, 2 MiB burst, full after a wake), shared by its sockets. An + overdraft closes the overdrawing socket (BACKPRESSURE, 1013). +- Cap of 4 sockets per device: a 5th connect closes that device's oldest socket with 1001 `device_socket_limit`, + then accepts. This runs before the vault-wide check, so a device at its cap can always reconnect. +- Five live clients on one device would evict each other forever, so the client backs off ≥ 30 s on + `device_socket_limit`. +- The vault-wide cap of 1000 (`429 stream_socket_limit`, `Retry-After: 1`) stays. +- *Why:* the limit must bind the writer, not the socket. Ghost sockets after a network switch are the usual 5th. + *Tests:* T-RATE-DEVICE, T-RATE-SOCKET, T-SOCKET-CAP-DEVICE. + +**H7 Retired-stream GC.** +- When `coversSeq == lastSeq`, the head UPDATE also sets `open=NULL, open_rows=0, tail_first=NULL, gc_seq=lastSeq`. + Sealed segments are all ≤ N and deleted as today; the head row stays. Cost: still 1 row (no indexed column in SET). +- Client note: sealed segments of a retired stream are only collected if the client writes a final checkpoint. +- Update §8. *Tests:* T-RETIRED-GC, T-PARTIAL-GC. + +**H8 minIntervalMs.** +- Default 1000 (`YAOS_STREAMS_GC_MIN_INTERVAL_MS` still overrides), echoed in `VAULT_READY.limits.groupCommit`. +- The idle flush waits `max(idleMs, lastCommitAt + 1000 − now)`; the max (1500 ms) and bytes (64 KiB) triggers stay. +- Small writes: ≤ 1 commit/s/vault, about 2–4 rows/s while typing, so 100k rows ≈ 7–14 h of continuous editing. +- Large writes are not bounded: the 64 KiB trigger still commits back to back (≤ 4/s/device at the rate gate). + Accepted, and stated in §11.5. +- *Tests:* T-MININTERVAL-READY, T-MININTERVAL-TIMING (≥ ~900 ms between back-to-back commits). + +**BASELINE** (unchanged behaviour, must stay green): T-OVERSIZE-1009, T-CKPT-MULTICHUNK (2.5 MB), T-TWO-SOCKETS, +T-HAPPY. + +--- + +## 5. Contract delta against relay-wire.md + +Classes: **additive** (new; old clients unaffected), **relaxation** (the server accepts or promises more), +**tightening** (it accepts or promises less), **removal** (gone). + +| § | Old → new | Class | Client action | +|---|---|---|---| +| 2.1 | Capabilities: keep `claimed`, `streams:1`, `attachments`, `maxBlobUploadBytes`, `serverVersion`. Drop `schemaVersion`, `protocolVersion`, `snapshots`, `recoveryJobs`, `settingsSync`, `semanticCanvas`, `bulkCreate`, `relayBodies`, `storageFormatVersion`, `snapshotFormatVersion`, `settingsFormatVersion`, `update*` | removal | none (pairing.ts reads only the kept five) | +| 2.2 | Claim body unchanged; the response drops `mobileSetupQrDataUrl` (G25, O12); already claimed → `409 already_claimed` | removal | none (the client never read it) | +| 2.3 | + `GET …/devices`, `DELETE …/devices/:deviceId`, `POST …/reset-streams`, `POST …/restore`; `/provision` → 404; `DELETE /operator/vaults/:id` takes `{"confirmVaultId"}` (D5) | additive / removal | none | +| 2.4 | Pairing code: "8–512 printable" → `^[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{32}$` after trim; malformed → `400 invalid_code`; unknown → `404 invalid_code` (was `unknown_code`) | tightening | none: `enrollHttpError` maps both codes; strip any client-only key part | +| 2.4 | Same `enrollmentRequestId` + deviceId + token hash → replay 200; same id, other body → `409 enrollment_request_conflict` (D3) | additive | none (pairing.ts already re-sends the identical body) | +| 2.4 | `202 authorization_fence_pending` → never sent | removal | none (the retry loop becomes dead code) | +| 2.4 | 200 body: exactly `host, deviceToken, vaultId, deviceId, deviceName, vaultGeneration` (D3) | removal | none | +| 2.5 | Pairing-code response `{pairingCode, expiresAt, purpose, obsidianUrl, mobileSetupUrl?}`; purpose `device` only (absent = device; else `400 invalid_purpose`) | tightening | none | +| 2.6 | Ticket wire unchanged; issued by the vault DO | none | none | +| 2.7 | Removed: `403 capability_denied`, `409 vault_`, `503 vault_draining`; `404 unknown_vault` → `401 unauthorized` (§2.2). `409 vault_generation_mismatch` gains `vaultEpoch`. + D8b: `503 restore_in_progress` with `Retry-After`, `409 restore_in_progress`, `503 restore_incomplete`, `400 invalid_restore_point`, `501 restore_unsupported` | removal / additive | none | +| 3.1 | `unclaimed` frame never sent (D4); DO-level `409 authority_superseded` and `403` at upgrade removed; `unauthorized` and `update_required` frames unchanged; 429 stays | removal | treat as `unauthorized` (already mapped) | +| 3.2 | `canWrite` always true; D6 constants; the `write_forbidden` note is removed; `groupCommit.minIntervalMs` 0 → 1000 | removal / tightening | read `limits`; set receipt timeouts ≥ maxMs + RTT | +| 4.3 | `STREAM_APPEND_REJECTED write_forbidden` removed. `VAULT_ERROR durability_failed` + `retryAfterMs` (ms, number) | removal / additive | when present, wait `retryAfterMs` before resending | +| 5.1 | Idle flush deferred to `lastCommitAt + minIntervalMs` | tightening | none | +| 5.4 | Dedupe scope (text below) | relaxation | none | +| 6, 7, 8 | Optional `epoch=`; mismatch or empty → `409 {"error":"vault_generation_mismatch","vaultEpoch"}` before any effect | additive | send it; on 409 switch to the returned epoch (new local DB) | +| 8 | GC text (below) | tightening | write a final checkpoint at `lastSeq` to retire a stream | +| 10 | 1001 also means "replaced by a newer socket of the same device" (reason `device_socket_limit`); 4403 only on revoke; + 1013 `restore_in_progress` (D8b) | additive | back off ≥ 30 s on `device_socket_limit` | +| 11.1 | Rate gate per socket → per device; + "Streams sockets per device: 4, a 5th evicts the oldest (1001 `device_socket_limit`)"; group commit row gains "1000 ms min interval" | tightening | one socket per vault per device | +| 11.2 | Permissions table and the 5 s authority cache removed; revoke shuts the gate at once (D7) | removal | none | +| 11.3 | `` → `<64 lowercase hex address>`; no hash check; `400 hash mismatch` removed; PUT overwrites; capability names gone | relaxation / removal | verify downloads client-side | +| 11.4 | Unchanged; the simulate route exists only with `YAOS_DEBUG_ROUTES=1` | none | none | +| 11.5 | + three identity tables; enroll 2 rows, revoke 1, ticket 0; + "the 64 KiB trigger still commits back to back" (H8) | additive | none | +| 13 | `YAOS_STREAMS` flag gone; the Node/Docker host gone | removal | none | +| 14 | Everything listed → 404, including "Still used": `me`, `members`, `devices` (vault-level), `invitations`, `device-links`, `principals/*`, `ownership/*`, `governance` | removal | none (`src/` calls none of them) | +| 15 | HTTP routes no longer pay a config-DO hop; receipt p50 rises toward 1 s for back-to-back commits | informational | none | +| 16 | `role` type stays `"owner" \| "member"`; only `"owner"` is sent | none | none | + +New §5.4 "Dedupe scope": + +> Duplicates are detected in the pending buffer (vault-wide by `(deviceId, clientFrameId)`, comparing stream and +> bytes) and against every committed row among the newest 4 MiB (stored row bytes) of the stream. Outside that +> window a resend is appended again with a new seq. The window covers more than one burst plus one max payload +> plus one commit batch, so every resend this section requires is deduped. The window is kept in memory only. A +> stream's first append in a runtime builds it in chunks, and that stream's frames wait in the pending buffer +> until it is installed: their receipts come later, and none is lost. + +New §8 "What survives": + +> When `N < lastSeq`, the open segment (the newest rows, under 64 KiB) survives, so readers just behind the +> checkpoint can still replay rows. When `N == lastSeq`, the open segment is dropped too and `gcSeq = lastSeq`. +> The stream keeps only its head and the checkpoint, and every read below `gcSeq` returns the checkpoint. Sealed +> segments are only collected by a checkpoint, so a retired stream needs a final checkpoint at `lastSeq`. + +--- + +## 6. Storage + +### 6.1 Vault DO schema + +The DDL runs once, at vault init, and never on a request path (T-PAIR-NOWRITE). The three stream tables are +unchanged from `streams/store.ts`: + +- `stream_head(stream PK, last_seq, ckpt_seq, gc_seq, tail_first, open_rows, open BLOB) WITHOUT ROWID` + index + `stream_head_last_seq(last_seq)`; +- `stream_segment(stream, first_seq, last_seq, rows, bytes, PK(stream, first_seq)) WITHOUT ROWID`; +- `stream_checkpoint(stream, chunk, covers_seq, bytes, PK(stream, chunk)) WITHOUT ROWID`. + +New: + +```sql +CREATE TABLE vault_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + vault_id TEXT NOT NULL, vault_generation TEXT NOT NULL, + ticket_key BLOB NOT NULL, -- 32 random bytes, never leaves the DO + created_at INTEGER NOT NULL, + pending_restore_id TEXT, -- D8b: set by prepare, erased by a real rewind + last_restore_id TEXT, -- D8b: step 2 skip, finish idempotency + last_restore_at INTEGER +) WITHOUT ROWID; +CREATE TABLE device ( + token_hash BLOB PRIMARY KEY, -- SHA-256(deviceToken) + device_id TEXT NOT NULL, device_name TEXT NOT NULL, + enrollment_request_id TEXT NOT NULL, enrolled_at INTEGER NOT NULL +) WITHOUT ROWID; -- no secondary index: revoke scans (a few rows) +CREATE TABLE pairing_code ( + code_hash BLOB PRIMARY KEY, -- SHA-256(secret) + purpose TEXT NOT NULL, expires_at INTEGER NOT NULL, used_at INTEGER, + used_request_id TEXT, used_device_id TEXT -- D3 replay key +) WITHOUT ROWID; +``` + +Probing an unknown vault reads `vault_meta`; "no such table" means unknown, cached in memory (init runs in the same +object, so the cache stays coherent). Rows read per runtime: meta 1 + devices N, then 0 per bearer auth. deviceId +uniqueness is checked in code in the same synchronous turn. Minting a code prunes rows expired over 24 h. + +### 6.2 Rows written per operation (cfRowModel rules) + +| Operation | Statements | CF rows | +|---|---|---| +| Vault init | INSERT meta (DDL not billed by the model; one-time) | 1 | +| Enroll | UPDATE pairing_code (used_*) + INSERT device | 2 | +| Enroll replay (D3) | none | 0 | +| Pairing / owner code | INSERT pairing_code (+1 per pruned row) | 1 | +| Ticket | none (key and devices in memory) | 0 | +| Revoke | DELETE device | 1 | +| Reset | DELETE H heads + S segments + C chunks; UPDATE meta | H+S+C+1 | +| Restore prepare (D8b) | UPDATE meta (marker; the rewind erases it) | 1 | +| Restore finish (D8b) | replace D device rows; DELETE P codes; UPDATE meta | ≈ D+P+1 | +| Commit | per touched stream: head UPDATE/INSERT (last_seq indexed) = 2; +1 per sealed segment | 2/stream + 1/seal | +| Checkpoint | DELETE old chunks (k₀) + INSERT new chunks (k₁) + DELETE collected segments (g) + UPDATE head (SET has no `last_seq`) | k₀+k₁+g+1 | +| Retired GC (H7) | same head UPDATE | +0 | +| Dedupe index (H2) | memory only | 0 | + +### 6.3 Config DO schema + +```sql +CREATE TABLE operator (id INTEGER PRIMARY KEY CHECK (id = 1), + key_hash BLOB NOT NULL, claimed_at INTEGER NOT NULL) WITHOUT ROWID; -- SHA-256(recovery key ≥ 32 chars) +CREATE TABLE session (token_hash BLOB PRIMARY KEY, expires_at INTEGER NOT NULL) WITHOUT ROWID; +CREATE TABLE vault (vault_id TEXT PRIMARY KEY, name TEXT NOT NULL, created_at INTEGER NOT NULL) WITHOUT ROWID; +CREATE TABLE restore_journal (vault_id TEXT PRIMARY KEY, restore_id TEXT NOT NULL, at INTEGER NOT NULL, + bookmark TEXT, devices BLOB, -- set after prepare; token hashes only + created_at INTEGER NOT NULL) WITHOUT ROWID; -- D8b; alarm while rows exist +``` + +`claimed` = the operator row exists. Rows written: claim 3 here (operator, vault, session) + 2 in the vault DO +(meta, code); login 1 (+1 per pruned session); logout 1; create vault 1 + 1 in the vault DO; delete vault 1 + the +vault DO's `deleteAll()` (not measured); restore 3 (insert, update, delete). + +--- + +## 7. Acceptance matrix + +"scratch-3" is today's `yaos-relay2-scratch-3` (legacy server, `YAOS_STREAMS=true`, no R2). BB results are +measured: `experiments/logs/conformance-scratch3-baseline-20261005T215906Z.json` and +`conformance-scratch3-flags-20261006T050159Z.json`; the revoke rows from `conformance-scratch3-revoke-20261006T051325Z.json`. +These three ran uncommitted suite versions, so their check names can differ from the committed suite (for example +T-ENROLL-BODY); the runner now records `+dirty` in that case. Second measurement: the same legacy commit under +local `wrangler dev` (`conformance-local-legacy-20261006T060730Z.json`) gives the same status on all 50 BB tests, so +local dev stands in for deployed in P2–P4; only timings differ (round trip ≈ 0.4 ms vs 80 ms). `*` marks a WB test: +it never runs against scratch-3, so the entry is a guess at today's code. BB files live in `e2e/conformance/tests/`, +WB files in `tests/server/` (WB file names are suggestions). + +| ID | What | Kind | scratch-3 | Where | +|---|---|---|---|---| +| T-HAPPY | enroll 2 devices, append b:/ns, receipts, provisional/committed, feed, read, checkpoint | BB | PASS | happy.ts | +| T-TWO-SOCKETS | two devices see each other's frames live | BB | PASS | socket.ts | +| T-OVERSIZE-1009 | binary > 1 MiB + 1 KiB closes 1009 | BB | PASS | socket.ts | +| T-CKPT-MULTICHUNK | 2.5 MB checkpoint round-trips (3 chunks) | BB | PASS | checkpoint.ts | +| T-ENROLL-200 | first enroll is 200 with the §2.4 fields, never 202 | BB | PASS | onboarding.ts | +| T-HOTPATH | zero config-DO calls on the 7 device paths | WB | FAIL* | hotpath.ts | +| T-PAIR-FORMAT | minted codes match the D3 regex, prefix = vaultId | BB | FAIL | onboarding.ts | +| T-PAIR-MALFORMED | malformed code → 400 invalid_code | BB | FAIL | onboarding.ts | +| T-PAIR-UNKNOWN-VAULT | unknown vault, and unknown secret on a known vault → 404 invalid_code | BB | FAIL (404 unknown_code) | onboarding.ts | +| T-PAIR-USED | second use → 409 used_code | BB | PASS | onboarding.ts | +| T-PAIR-NOWRITE | unknown vault: 0 rows written, no DDL | WB | FAIL* | pairing.ts | +| T-ENROLL-REPLAY | identical retry → 200, same deviceId, vaultId, generation; token works | BB | PASS | flags.ts | +| T-ENROLL-CONFLICT | same id, other deviceId or token → 409 enrollment_request_conflict; original token works, other 401 | BB | PASS | flags.ts | +| T-ENROLL-BODY | exactly the six client fields | BB | FAIL (legacy principalId, extra fields) | flags.ts | +| T-TICKET-CROSS-VAULT | vault A ticket on vault B → 1008 unauthorized | BB | PASS | onboarding.ts | +| T-TICKET-BAD | tampered, garbage or missing ticket → 1008 unauthorized | BB | PASS | onboarding.ts | +| T-UNKNOWN-VAULT-401 | valid bearer on an unknown vault → 401 on ticket, pairing-code, feed, read, checkpoint | BB | PASS | flags.ts | +| T-LEGACY-404 | the D5 legacy list → 404 | BB | FAIL | operator.ts | +| T-CONSOLE-ROUTES | `GET /`, `GET /mobile-setup` → 200 HTML; CORS preflight not 404 | BB | PASS | flags.ts | +| T-PROVISION-404 | `POST …/provision` → 404 | BB | FAIL (200) | flags.ts | +| T-VAULT-DELETE-CONFIRM | missing or wrong confirmVaultId → 400; match → 200, sockets 1001, bearer 401 | BB | FAIL (409 destroy_confirmation_required) | flags.ts | +| T-DEVICES-LIST | deviceId and deviceName, no token material | BB | FAIL (404) | operator.ts | +| T-READY-SHAPE | D6 constants, every §3.2 field | BB | PASS | socket.ts | +| T-REVOKE-GATE | no frame the revoked socket sends after the revoke response commits (incl. after idle) | BB | PASS | revoke.ts | +| T-REVOKE-SILENCE | the revoked socket gets no data-bearing message after the revoke response | BB | PASS | revoke.ts | +| T-REVOKE-BUFFER | frames buffered for group commit at revoke never commit, no receipt | BB | FAIL (23/23 committed) | revoke.ts | +| T-REVOKE-4403 | error frame and close frame 4403 each < 1 s; TCP end recorded | BB | FAIL (route; legacy owner route: 207 / 211 ms deployed, 6 / 10.5 ms local; TCP end ≈ 10 s in both) | operator.ts | +| T-REVOKE-401 | revoked bearer → 401 on ticket, feed, read, pairing-code | BB | FAIL (route; legacy: all 401) | operator.ts | +| T-RESET | new epoch, empty streams, 1001 closes, devices kept | BB | FAIL (its restore probe: 404 on legacy, deployed and local; 501 without PITR applies to the rewrite only) | operator.ts | +| T-RESTORE-RESUME-WB | fake PITR port, crash after each step; resume ends with snapshot devices, no codes, new epoch, no journal row | WB | FAIL* | restore.ts | +| T-RESTORE-MANUAL | restore to T: content rewound, devices as of the request, codes gone, epoch rotates (deployed only) | manual | n/a | manual | +| T-EPOCH-MISMATCH | wrong or empty epoch → 409 + vaultEpoch, no effect | BB | FAIL | checkpoint.ts | +| T-EPOCH-MATCH | matching epoch → normal | BB | PASS | checkpoint.ts | +| T-EPOCH-ABSENT | no param → normal | BB | PASS | checkpoint.ts | +| T-BLOB-OPAQUE | non-SHA 64-hex address PUT/GET/exists, overwrite | BB | SKIP (no R2) | misc.ts | +| T-BLOB-KEY-RESET | a blob survives reset-streams byte-identical and in `exists` | BB | SKIP (no R2) | flags.ts | +| T-BLOB-UNAVAILABLE | 503 attachments_unavailable, attachments=false | BB | PASS | misc.ts | +| T-CODEC-UTF8 | invalid UTF-8 → 1008 | BB | PASS | socket.ts | +| T-CODEC-SURROGATE | `ED A0 80` → 1008 | BB | PASS | socket.ts | +| T-CODEC-OVERLONG | `C0 AF` → 1008 | BB | PASS | socket.ts | +| T-CODEC-NONMINIMAL | `0x82 0x00` → 1008 | BB | FAIL | socket.ts | +| T-CODEC-TRAILING | trailing byte → 1008 | BB | PASS | socket.ts | +| T-CODEC-VALID | `"b:é✓😀"` accepted | BB | PASS | socket.ts | +| T-DEDUPE-LARGE | 3×600 KiB, reconnect, resend → deduped, original seqs | BB | FAIL | commit.ts | +| T-DEDUPE-SMALL | 400×4 KiB resend → deduped | BB | FAIL | commit.ts | +| T-DEDUPE-CONFLICT | same id, other bytes → client_frame_id_conflict + seq | BB | PASS | socket.ts | +| T-DEDUPE-COLD-WB | the same after a runtime restart; ≤ 65 rows read | WB | FAIL* | dedupe.ts | +| T-DAILY | simulated limit → VAULT_ERROR cf_daily_limit; checkpoint 503 | BB | SKIP (legacy's switch is `YAOS_TEST_ONLY_DEBUG_ROUTES`, unset; the rewrite uses `YAOS_DEBUG_ROUTES=1`) | misc.ts | +| T-COMMIT-DAILY-WB | the relay types the daily limit itself | WB | PASS* (same wire) | relay-commit.ts | +| T-COMMIT-RETRY-WB | durability_failed + retryAfterMs, backoff, reset | WB | FAIL* | relay-commit.ts | +| T-SOCKET-CACHE-WB | attachment parsed once per socket per runtime; wake rebuild skips revoked devices | WB | FAIL* | socket-cache.ts | +| T-RATE-DEVICE | two sockets of one device share one bucket → 1013 | BB | FAIL | commit.ts | +| T-RATE-SOCKET | one socket overdraft → BACKPRESSURE + 1013 | BB | PASS | commit.ts | +| T-SOCKET-CAP-DEVICE | 5th socket evicts the oldest with 1001 | BB | FAIL | commit.ts | +| T-RETIRED-GC | ckpt at lastSeq → gcSeq=lastSeq, read = checkpoint only | BB | FAIL | checkpoint.ts | +| T-PARTIAL-GC | ckpt below lastSeq → open survives | BB | PASS | checkpoint.ts | +| T-MININTERVAL-READY | VAULT_READY minIntervalMs 1000 | BB | FAIL | socket.ts | +| T-MININTERVAL-TIMING | back-to-back commits ≥ ~900 ms apart | BB | FAIL | socket.ts | +| T-ROWS-WB | every §6.2 row count under cfRowModel | WB | n/a | rows.ts | + +--- + +## 8. Flag log and open items + +Review flags: + +- F1 → accepted (mandatory), now in D3. +- F2 → accepted, now in D3. +- F3 → accepted (mandatory), now in D9. +- F4 → accepted, now in D4. +- F5 → accepted, now in §2.2 and D5. +- F6 → accepted, now in D5. +- F7 → accepted, now in D5. +- F8 → accepted, now in D5. +- F9 → replaced by the new D8b design (approved 2026-10-06). +- F10 → accepted, now in D3. +- F11 → accepted, now in §2.2. +- F12 → accepted, now in H2. +- F13 → accepted, now in H6. +- F14 → accepted, now in H8. +- F15 → decided and done on this branch, now in §2.3. +- F16 → accepted, now in D8a. +- F17 → flipped (buffered frames of a revoked device are dropped), now in D7. +- F18 → accepted, now in H2. + +Raised during the fold, now resolved: + +- O1 D3 replay scope → key = code + `enrollmentRequestId` + deviceId + token hash; `deviceName` ignored; a revoked + device → `409 used_code`. Now in D3. +- O2 D3 enroll body → exactly the six fields `readEnrollment` reads; no D6 constants. Now in D3. +- O3 D7 wake rebuild → kept: H5's rebuild skips and closes 4403 sockets whose device is gone. +- O4–O8 D8b gaps (authority drift, stuck flag, writes in the window, concurrent restores, loose ends) → the D8b + rewrite: config-DO-run steps, alarm resume, `pending_restore_id` marker, re-rewind on writes, authority freeze, + flag TTL, PK reservation, synchronous 200, error codes. + +P1 gap calls (accepted by the coordinator; each is marked `DECISIONS-GAP` in code): + +- G1 No `update-manifest.json` in the release (§1 removes update-metadata). +- G2 Config DDL runs on the singleton's first construction. +- G3 `shared/socketLiveness.ts` trimmed to what the relay uses (semanticEpoch is gone). +- G4 `version.ts` keeps only `SERVER_VERSION`. +- G5 `/enroll` body cap 64 KiB (same as claim). +- G6 `:deviceId` path segment uses the legacy enroll deviceId pattern. +- G7 `GET /` and `GET /mobile-setup` are placeholders until P4. +- G8 Unexpected Worker failure → `500 internal_error`. +- G9 Non-JSON enroll body → `400 invalid_code`. +- G10 The Worker checks `streamsVersion` before forwarding; the ticket is checked in the vault DO. +- G11 Malformed blob address → `400 invalid_address`. +- G12 `compatibility_date = "2026-04-07"` (`web_socket_auto_reply_to_close` on); wrangler 4.147.0. + +P2 and P4 gap calls (accepted; marked `DECISIONS-GAP` in code). G7 is closed: router.ts serves the P4 pages. + +- G13 Operator login limiter: in memory, 20 failures/min → `429 too_many_attempts` (D3's numbers). +- G14 A vault not in the registry → `404 unknown_vault` on operator routes. +- G15 CSRF: every operator write needs a same-origin `Origin`; JSON writes also need the Content-Type; GETs need + neither (SameSite=Strict carries them). +- G16 A frame dropped by revoke tells holders `STREAM_PROVISIONAL_DROPPED` with reason `commit_failed`. +- G17 Enroll replay ignores code expiry (it needs the original deviceToken, so it grants nothing new). +- G18 Enroll of a deviceId that is enrolled under another code → `409 device_exists`. +- G19 A valid ticket on a non-upgrade request → `426`. +- G20 Vault names: 1–80 chars, else `400 invalid_name`. +- G21 The session cookie is always `Secure` (browsers treat `http://localhost` as secure). +- G22 Claim failing after the config write → `503 claim_incomplete`, with the session cookie set. +- G23 Vault-delete R2 purge: at most 20 batches of 1000 per request, then `503 purge_incomplete` (retry). +- G24 The D3 enroll limiter counts 404, 410 and `409 used_code`; malformed bodies don't count. +- G25 Operator response shapes (the console reads only these): state `{vaults:[{vaultId,name,createdAt}], + pendingRestores:[{vaultId,at}]}`; devices `{devices:[{deviceId,deviceName,enrolledAt}]}`; owner-code adds + `mobileSetupUrl`. Neither claim nor owner-code carries a QR; the console draws it (O12). +- G26 The console generates the recovery key in the page and sends `/claim` only after "I have saved it" is ticked. +- G27 The entry module (`worker.ts`) exports only the fetch handler and the DO classes: workerd treats every named + export as an entrypoint and refuses to start on a constant. The route table is `router.ts`; a WB test guards it. + +P3a gap calls (accepted; marked `DECISIONS-GAP` in code): + +- G28 The daily-limit up-front refusal uses the last classified kind. +- G29 H2 build steps are 16,384 rows; the window read is charged one unit per 256 B. +- G30 Cold builds run one at a time, in queue order (a cold stream can wait behind a large build). +- G31 Build steps run once per incoming message and once per timer flush. A forced flush builds inline; a failed + flush fails the frames it held. +- G32 The per-device rate bucket survives reconnects and is deleted only on revoke. +- G33 While a stream is gated, seq order across streams can differ from arrival order; per stream it is kept. +- O10 (accepted risk) Timer-driven build steps share the CPU budget of the last incoming message, since only + requests and messages reset it (DO limits page). One message and then silence on a worst-case stream runs + ≈ 10 steps (≈ 13 ms) in one window, over Free's 10 ms; the platform tolerates infrequent overruns ("built-in + flexibility", Workers limits page). A vault-DO alarm per step would give each a fresh budget; add it only if + P5 shows `exceededCpu`. P5: none, but scratch-3's account is Workers Enterprise, so Free's 10 ms is not enforced + there and cannot show up. Worker CPU p50 0.6 ms (the old server: 2.4 ms). Over 10 ms (GraphQL + `workersInvocationsAdaptive`): claim 64 ms and owner-code 7–46 ms in the Worker, both rendering the setup QR + (moved to the console page, O12; Worker max since: 6.1 ms), and VaultDO 4 MiB stream reads 21–24 ms. Still open + until a Free account is measured. +- O9 → resolved: a daily-limit failure on any route, including one thrown by a DO RPC, is `503 cf_daily_limit` + with Retry-After (router.ts:187-188; WB in tests/server/reset.ts). Revoke under the latch changes nothing and + says so; the device keeps its read access until the reset (≤ 24 h). Kept: a gate shut in memory only would not + survive eviction. + +P3b gap calls (accepted; marked `DECISIONS-GAP` in code): + +- G34 `POST /vault/:id/blobs/exists` body cap 64 KiB → `413 body_too_large` (50 addresses are ≈ 3.5 KiB). +- G35 A blob GET is always `application/octet-stream` with `nosniff` and `no-store`: blobs share the console's + origin, so an uploaded HTML or SVG body never renders. +- G36 A restore body that is not a JSON object counts as `{}` (no `at` → `400 invalid_restore_point`). +- G37 The restore alarm re-arms itself while journal rows remain, after the newest row's age clamped to 30 s..1 h + (about doubling per alarm). The age is read from `created_at`, so an eviction does not reset the backoff (P5: an + in-memory counter re-armed every 30 s on scratch-3). +- G38 One run rewinds at most 3 times, then `503 restore_incomplete`; the alarm carries on. +- G39 `at` is a date-time with optional seconds, ≤ 3 fraction digits and an explicit zone (`Z` or ±hh:mm). +- G40 `at` is validated even when a restore is pending; the pending one then resumes with its journaled `at`. +- G41 A vault that answers `unknown_vault`, or whose journal row vanished mid-run, ends the run `404 unknown_vault`; + `restore_unsupported` and `invalid_restore_point` from the vault drop the row. +- G42 An `at` before the vault was created → `400 invalid_restore_point`, with no PITR call. So does an `at` that + Cloudflare's PITR history does not reach (P5: "Requested time is before this database existed.", or "This + database has no history." before the first snapshot): nothing was done to the vault, and the journal row is + dropped. +- G43 Finish reads "`pending_restore_id` still set" as "equals this restoreId": a restore in flight at T leaves its + own marker in the rewound state, and rewinding never clears that one. +- G44 Enroll's restore 503 runs before the code is read, so a malformed body also gets it. +- G45 The device routes' restore 503 runs after auth and D8c (no oracle); blob and debug routes count as device + routes. +- G46 An upgrade during a restore is refused after the ticket check, with the error frame and 1013 + `restore_in_progress`; a non-upgrade request with a valid ticket gets the 503. +- G47 TEST-ONLY simulate-daily-limit: `enabled` must be a boolean (else `400 invalid_request`); `200 {ok, enabled}`. +- O11 (accepted risk) Step 0 issues the journal INSERT and `setAlarm` in one synchronous turn (config/restore.ts + :126-130), so the platform should commit them together (write coalescing); not verified on the platform. A + daily-limit failure injected at `setAlarm` alone (WB) leaves a row without an alarm: the vault is untouched + (step 1 never ran), but its authority actions answer `409 restore_in_progress` until the operator presses restore + again, which resumes the journaled `at`. + +P5 findings: + +- O12 → resolved (the user's call, 2026-10-06: the console draws it). The setup QR was rendered in the Worker + (`qrcode-generator` `make()`: 8 ms cold, 2.6 ms warm on a laptop for the 195-character URL, version 10), which made + claim and owner-code the only Worker routes over Free's 10 ms CPU (O10). Now the console page inlines + qrcode-generator 2.0.4's browser build (`dist/qrcode.js`, 56 KB) verbatim as its first nonce script; a wrangler + Text rule bundles it as text, and the Node tests alias the import to the same file (tests/mocks/qrcodeScript.ts). + The page encodes setupQr.ts's mobile setup URL on `location.origin` as the SVG the Worker sent. Claim and + owner-code drop `mobileSetupQrDataUrl` (G25); the console was its only reader and the client never read it. WB: + tests/server/console.ts matches the drawn modules to the library's for that URL. Headless Chrome on local + `wrangler dev` (d51c3ae): the encoder runs under the CSP, and jsQR decodes both QRs (claim, owner-code) to the + exact URL; no page error or CSP violation. On scratch-3 (version 4de93251), the same check signs in and decodes 9 + owner-code QRs to the exact URL. The one failed request is the 401 from `GET /operator/state` before sign-in, + which shows the sign-in form (console.ts:150). Worker CPU in the minute of those 9 owner-code calls and 10 other + console requests: max 2.6 ms, p50 1.6 ms (was 7–46 ms). Claim cannot run again on claimed scratch-3. Its Worker work + is now what login and owner-code do: RPCs, a cookie and a URL (router.ts:261-304). Suite on 4de93251: + `conformance-scratch3-o12-20261006T132806Z.json`, 48 pass, 0 fail, the same 2 SKIP; Worker CPU over the run max + 6.1 ms, p50 0.5–0.9 ms per minute. +- O13 A refused upgrade (relay-wire §3.1: accept, error frame, close, 101; vault/cloudflare.ts) logs as a VaultDO + `scriptThrewException` "Network connection lost" on scratch-3. Clients get the frame and the close code (the + refusal rows pass); the cost is noise in error analytics. +- O14 Two smoke receipts took 1525 and 1660 ms (the next ones 457 and 373 ms) with the same delay at both peers, so + the commit itself was late. Unexplained; untested guess: frames held during the dedupe index build and + rescheduled (relay.ts:684-689). p90 only; no check failed. +- O15 (accepted; platform) A restore lands on Cloudflare's latest PITR snapshot at or before `at`, not at `at`. + Measured on scratch-3 (one frame every 2 s, restored every 2 s): snapshots about a minute apart, the first 45–55 s + after the vault's init, plus one just before each rewind. So a restore can drop up to a minute of content before + `at`, and a vault has no restore point in its first minute. The `200` cannot say which snapshot (a bookmark + carries no time); the console's restore text, confirm and result say "the last snapshot at or before". + A restore to a point before an earlier restore works (history survives a restore). +- O16 (accepted risk) A step-1 error that is permanent but not classified (G42 lists the two PITR ones seen) keeps + the journal row: `503 restore_incomplete`, the alarm at ≤ 1 h, and the vault's revoke, owner-code and reset + frozen (`409 restore_in_progress`) until the vault is deleted; there is no cancel. Seen once on scratch-3, from + the unclassified "before this database existed", now classified. + +## 9. Work plan + +| Phase | Work | Exit criterion | +|---|---|---| +| P0 Conformance baseline | BB suite in `e2e/conformance/` for every BB row of section 7; run against scratch-3 and local `wrangler dev` | The suite runs end to end; scratch-3 results recorded, with every difference from the section 7 guesses explained | +| P1 Host skeleton | New Worker router, vault DO and config DO classes; streams core behind ports; legacy `server/src` deleted; README "Cloudflare only". The Node host, Docker and `packages/cli` are already gone on this branch, with the harness adapter in `tests/server/helpers/` | Ported streams WB tests green; T-LEGACY-404 and T-HOTPATH harness in place; `tsc`/lint clean | +| P2 Identity | Claim, login and sessions, vaults, owner code, D3 codes, enroll (replay, D3), D4 tickets, devices list and revoke (D7 gate) | D2–D7 BB + BASELINE green on local dev; T-PAIR-NOWRITE, T-HOTPATH, T-ROWS-WB (identity rows) green | +| P3 Hardening | H1–H8, D8a/D8c, D9 (`v//
` keys); D8b | Every BB row except T-RESTORE-MANUAL green locally (blob rows with local R2); all WB green; H2 cold-scan CPU and D8a DROP+CREATE billing measured | +| P4 Console | `GET /` page and `GET /mobile-setup` (D5) | Manual run on local dev: claim → vault → QR → enroll → revoke → reset; no external assets | +| P5 Deploy over scratch-3 | The coordinator deploys: migration v3 deletes the old DO classes, then v4 adds the two new ones (one migration doing both failed with `10067` at the 500-namespace cap: the API counts the new classes before the deletions) | The full suite is green on scratch-3 except the documented SKIPs; T-RESTORE-MANUAL done once by hand; §15 latencies re-measured | + +Status 2026-10-06: P0–P3 done. Combined local run at 15e9bf3: `conformance-local-final-20261006T105356Z.json`, +49 pass, 0 fail, 1 SKIP (T-BLOB-UNAVAILABLE needs an unbound bucket; it passes in +`conformance-local-p3b-nobucket-20261006T104800Z.json`). P4 manual run done on local dev (Chrome, desktop and 375 px): claim → owner code with +QR and `obsidian://` link → enroll (replay 200, reuse `409 used_code`) → revoke from the Devices list (error frame +`authority_superseded` +25 ms, list empty, token `401`) → reset (device kept, socket 1001, old epoch `409`) → restore +(`501 restore_unsupported` message) → vault create and delete → sign out and in → `/mobile-setup` (fragment dropped, +`connect-src 'none'` blocks fetch, foreign host and bad code refused). No external request. Fixed on the way: `.mono` +inputs overflowed the card at 375 px; the `invalid_restore_point` text now names the vault's creation (G42). + +Status 2026-10-06 (P5): deployed over scratch-3 in two steps (v3, then v4; §9 row). Suite at c754d59: +`conformance-scratch3-p5-20261006T113415Z.json`, 47 pass, 1 fail, 2 SKIP (T-BLOB-UNAVAILABLE: the bucket is bound; +T-DAILY: no debug route). The fail, T-CODEC-VALID ("timeout waiting for VAULT_READY"), was the network: the Worker +logged `responseStreamDisconnected` 40 ms in and the vault DO's fetch `canceled` after 1 ms, on a client behind +WARP; three re-runs pass (`conformance-scratch3-codecvalid-rerun{1,2,3}-*.json`). e2e/relay/smoke.ts, with `Origin` +on operator writes (G15), 31/31 three times (`client-e2e-smoke-deployed-new-{1,2,3}-*.json`). §15 p50 medians, old +deployed → new: ticket 107 → 91; socket connect 317 → 170; peer PROVISIONAL 57 → 61; own receipt 370 → 989; peer +COMMIT_NOTICE 357 → 994; peer COMMITTED 395 → 374; ping 67 → 64; 12 × 8 KiB burst 467 → 1150; feed 249 → 97; read +page 238 → 93; checkpoint put 266 → 117. The three slower rows are H8: smoke sends each append right after the last +receipt, inside the 1 s `minIntervalMs`; its first sample, from idle, is 372 ms (old 405). relay-wire §15 (the text +after the table) and :163 (`minIntervalMs: 0`) are updated in the PR into client-remake, with :56 (O12). +T-RESTORE-MANUAL failed first: PITR refused T, 11 s after the vault's init, and the runner retried forever with the +vault's authority frozen; fixed in 55cfd32 and f9ce3a1 (G37, G42, O15). Re-run on 55cfd32 (version 243e812e) with T +and T2 each 75 s after the writes they must keep and the vault's first snapshot awaited: 42 pass, 0 fail +(`restore-manual-scratch3-20261006T125029Z.json`). Final suite at f9ce3a1 (version 249ab495): +`conformance-scratch3-p5-final-20261006T125917Z.json`, 48 pass, 0 fail, the same 2 SKIP. P5 is done. CPU: O10, +O12. Open: O13, O14, O16 (O12 resolved after P5). diff --git a/e2e/conformance/README.md b/e2e/conformance/README.md new file mode 100644 index 00000000..a179d705 --- /dev/null +++ b/e2e/conformance/README.md @@ -0,0 +1,102 @@ +# YAOS relay conformance suite + +This is a black-box suite for the streams relay. It checks the wire contract (`docs/client-remake/relay-wire.md`) and +the server-remake decisions (D2–D9, H1–H8). It runs against any deployed or local relay over HTTPS and WebSocket only. +It is standalone: plain `node` (≥ 22.18, which strips TS types natively), no npm deps, and no imports from `src/`, +`server/` or `legacy-src/`. + +## Running + +```sh +node e2e/conformance/run.ts --host https://.workers.dev --label