diff --git a/.dev.vars.example b/.dev.vars.example index efdcebb..7c37ed9 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -1,10 +1,7 @@ -# Wrangler does not read .env, so `pnpm preview` needs the same values here. -# Copy to .dev.vars and fill in. -# -# The Convex deployment the previewed build talks to. `pnpm dev:backend` puts -# this in .env.local; copy the same value here. -NEXT_PUBLIC_CONVEX_URL= - -# See .env.example — same key, same warning: back it up, and never set it as a -# Convex environment variable. +# Copy to .dev.vars. Never commit real credentials or the existing master key. +AUTH_URL=http://localhost:3000 +AUTH_SECRET= +AUTH_GITHUB_ID= +AUTH_GITHUB_SECRET= +# Base64-encoded 32-byte key. Production must keep its existing value forever. ENCRYPTION_MASTER_KEY= diff --git a/.env.example b/.env.example index 05203a1..dd0a4e7 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,3 @@ -# Copy to .env for local development with `pnpm dev` (Node runtime). -# For `pnpm preview` / `pnpm deploy` (Cloudflare Workers runtime), the same -# values must live in .dev.vars instead — wrangler does not read .env. -# In production, set them as Worker secrets: wrangler secret put -# -# NEXT_PUBLIC_CONVEX_URL is NOT listed here: `pnpm dev:backend` writes it (and -# CONVEX_DEPLOYMENT) into .env.local for you. Without a deployment URL, sign-in -# and cloud storage are unavailable; only the device-bound guest workspace is -# available. - -# Master key (KEK) that wraps every per-user data-encryption key. It must be -# 32 random bytes in base64 — generate one with: openssl rand -base64 32 -# Deliberately kept on Cloudflare and NEVER set as a Convex environment -# variable: that separation is what stops the encrypted records and the key -# that opens them from living with the same provider. -# Losing it makes all encrypted cloud records unrecoverable: back it up in a -# password manager. If it is unset, the app syncs in plaintext (cryptoVersion 0). -ENCRYPTION_MASTER_KEY= +# Runtime configuration belongs in .dev.vars; see .dev.vars.example. +# Keep remote Images/AI disabled when testing without Cloudflare credentials. +KLIPCODE_REMOTE_BINDINGS=false diff --git a/.env.production b/.env.production index 7a3478f..b2bd9d5 100644 --- a/.env.production +++ b/.env.production @@ -1,21 +1,2 @@ -# Production build values. Committed on purpose. -# -# NEXT_PUBLIC_* is inlined into the client bundle at BUILD time, so it has to be -# present when the build runs — setting it as a Worker runtime variable does -# nothing, the bundle is already compiled by then. Cloudflare Workers Builds -# runs `opennextjs-cloudflare build` on its own runners, where neither -# .env.local nor the `convex deploy --cmd` wrapper in `pnpm deploy` exists, so -# without this file the production build compiles with no deployment URL and the -# app silently falls back to a device-only guest workspace. -# -# Keeping it here rather than in the Cloudflare dashboard means the value is -# reviewable, versioned, and cannot drift out of sync with the deployment. -# -# This is not a secret: the URL ships inside the client bundle either way, and -# access is gated by the identity checks inside each Convex function, never by -# the URL being unknown. The actual secret (ENCRYPTION_MASTER_KEY) stays a -# Cloudflare Worker secret and is never committed. -# -# .env.local wins over this file, so local builds keep pointing at the dev -# deployment; only builds without a .env.local (i.e. CI/Cloudflare) use it. -NEXT_PUBLIC_CONVEX_URL=https://befitting-badger-749.convex.cloud +# The cloud backend is same-origin and configured through runtime D1 bindings. +# No deployment URL or credentials are embedded in the browser bundle. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df4e9f5..f6ab409 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ jobs: - name: Test run: pnpm test + - name: Test migration importer + run: python3 scripts/test-migrate-convex-to-d1.py + - name: Build run: pnpm build diff --git a/README.md b/README.md index 08bb2e4..8298b49 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ guest workspace that stays on the device you are using. - Write and preview Markdown documents in a rich TipTap editor. - Generate a name for an untitled snippet with Workers AI (Llama 3.2 3B Instruct) when signed in. -- Sync the library through Convex after GitHub sign-in. Cloud records are - encrypted per user when the Cloudflare Worker secret `ENCRYPTION_MASTER_KEY` - is set; without it, records are synced in plaintext (`cryptoVersion 0`). +- Sync the library through Cloudflare D1 after GitHub sign-in. Cloud records + are encrypted per user using the existing `ENCRYPTION_MASTER_KEY` Worker + secret. Missing keys fail closed rather than uploading plaintext. A device downloads the whole library once, then only what changed since its - last sync (a server-clock cursor), which keeps Convex database I/O small. + last sync (a server-clock cursor), which keeps D1 database I/O small. - Use the app in English or Spanish, in light or dark theme, with keyboard shortcuts and a search palette. A service worker caches the app shell so the UI still loads offline. @@ -28,8 +28,9 @@ guest workspace that stays on the device you are using. ## Tech stack Next.js 16 (App Router, React 19) and Tailwind CSS 4, deployed as a Cloudflare -Worker through OpenNext. Convex provides the backend, the database, and GitHub -authentication; local snippet storage uses IndexedDB via Dexie. +Worker through OpenNext. D1 stores the workspace and Auth.js database sessions; +GitHub provides authentication. Local snippet storage uses IndexedDB via Dexie. +The private analytics console uses the same D1 database without a VPS. ## Requirements @@ -47,23 +48,25 @@ pnpm dev Open . This is enough to work on the guest workspace. -To develop cloud sync and authentication, run the Convex development backend in -a second terminal before starting the app: +To develop cloud sync, copy [.dev.vars.example](.dev.vars.example) to +`.dev.vars`, fill in a development OAuth app and encryption key, then run: ```bash pnpm dev:backend pnpm dev ``` -`pnpm dev:backend` writes the development deployment values to `.env.local`. -For encrypted cloud records, copy [.env.example](.env.example) to `.env` and set -`ENCRYPTION_MASTER_KEY`. GitHub sign-in also requires a GitHub OAuth app and -these Convex environment variables: +`pnpm dev:backend` applies D1 migrations locally. The GitHub callback is +`http://localhost:3000/api/auth/callback/github` for development and +`https://klipcode.com/api/auth/callback/github` for production. Use separate +OAuth applications for these environments. Missing OAuth credentials leave +the guest workspace available. -```bash -pnpm exec convex env set AUTH_GITHUB_ID -pnpm exec convex env set AUTH_GITHUB_SECRET -``` +Production uses `DB` as a D1 binding and Worker secrets `AUTH_SECRET`, +`AUTH_GITHUB_ID`, `AUTH_GITHUB_SECRET`, `AUTH_URL` and +`ENCRYPTION_MASTER_KEY`. Never replace the production master key: existing +wrapped keys and ciphertext depend on it. `pnpm deploy` applies versioned D1 +migrations before building and deploying the Worker. ## Checks and scripts @@ -82,7 +85,8 @@ CI runs all of the above on every push to `main` and on every pull request. ## Further reading - [Engineering audit](docs/audit/engineering-audit.md) -- [Convex backend](convex/) +- [D1 backend](src/server/) +- [Migration and rollback](docs/d1-migration.md) - [Environment example](.env.example) KlipCode is released under the [MIT License](LICENSE). diff --git a/convex.json b/convex.json new file mode 100644 index 0000000..a205825 --- /dev/null +++ b/convex.json @@ -0,0 +1,3 @@ +{ + "functions": "legacy/convex" +} diff --git a/convex/auth.ts b/convex/auth.ts deleted file mode 100644 index 0fc66c6..0000000 --- a/convex/auth.ts +++ /dev/null @@ -1,9 +0,0 @@ -import GitHub from "@auth/core/providers/github"; -import { convexAuth } from "@convex-dev/auth/server"; - -// GitHub is the only provider, matching the single "Sign in with GitHub" button -// in the aside. Credentials come from the Convex deployment env -// (AUTH_GITHUB_ID / AUTH_GITHUB_SECRET), never from the client bundle. -export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({ - providers: [GitHub], -}); diff --git a/docs/audit/engineering-audit.md b/docs/audit/engineering-audit.md index 7e1c319..218e5d1 100644 --- a/docs/audit/engineering-audit.md +++ b/docs/audit/engineering-audit.md @@ -1,5 +1,8 @@ # Engineering Audit +> Historical audit of the Convex architecture. The current D1 architecture and +> provider-separation change are documented in [D1 migration](../d1-migration.md). + **Audit date:** 2026-08-07 **Scope:** application code, Convex functions, local persistence, cloud sync, authentication, API routes, dependency graph, accessibility-sensitive UI paths, and a production-style browser smoke test. diff --git a/docs/d1-migration.md b/docs/d1-migration.md new file mode 100644 index 0000000..20dbff0 --- /dev/null +++ b/docs/d1-migration.md @@ -0,0 +1,110 @@ +# D1 migration + +The application and analytics console use Cloudflare Workers and the `klipcode` +D1 database. Production request paths have no Convex calls or VPS dependency. +GitHub accounts retain their existing user ids, so IndexedDB ownership and R2 +image paths remain valid. Ciphertext and wrapped DEKs are copied without +re-encryption. The master key stays the existing `klipcode` Worker secret. + +## Data and sessions + +- D1 workspace reads and writes include the authenticated owner in every query. +- Browser writes require the app's origin and expected account id; a cookie + change during sync cannot upload ciphertext into another account. +- Auth.js issues HTTP-only, Secure, SameSite cookies. Only hashes of opaque + session tokens are stored in D1. GitHub numeric identity ids resolve to the + imported users. Existing Convex sessions require GitHub sign-in again. +- Writes use indexed primary keys, last-write-wins timestamps and atomic D1 + batches. SQL triggers maintain deletion logs and analytics metadata. +- Pulls use a server-clock cursor, a one-minute overlap and a 30-day deletion + window. Old cursors and the pre-migration backend epoch force a full pull. +- Encryption failures stop sync and retain dirty local records for retry. +- D1 metadata and Worker secrets share a provider. Database exports still cannot + decrypt records alone, but this no longer provides separation of providers. + +## Repeatable copy + +Keep all exports and generated SQL outside Git, in a protected directory on +persistent storage or an external backup. Both contain private account data. +First test against local D1, then the remote database before routing app traffic. + +```bash +pnpm exec convex export --prod --path /protected/convex.zip +python3 scripts/migrate-convex-to-d1.py /protected/convex.zip \ + --analytics /protected/old-analytics-snapshot.json --sql /protected/import.sql +pnpm exec wrangler d1 migrations apply DB --remote +pnpm exec wrangler d1 execute DB --remote --file /protected/import.sql +python3 scripts/migrate-convex-to-d1.py /protected/convex.zip --verify +``` + +For the final frozen copy, use prepared REST batches rather than another bulk +SQL import. Cloudflare returned `D1_RESET_DO` on the large final SQL file during +the production migration. Prepared parameters avoid oversized SQL statements; +unchanged workspace records are skipped to preserve edit history and reduce +writes. The script refuses this path if D1 already contains login sessions. +Use a D1 API token from the environment or an existing protected Wrangler OAuth +configuration; never put a token in a command argument. +The copy script requires Python 3.11 or newer. Run its isolated SQLite/API tests +with `python3 scripts/test-migrate-convex-to-d1.py`. + +```bash +python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip \ + --apply-remote --source-frozen \ + --account-id YOUR_ACCOUNT_ID --database-id YOUR_DATABASE_ID \ + --oauth-config /protected/wrangler/config/default.toml +python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip --verify +``` + +The importer verifies every user id, provider identity, ciphertext field and +wrapped key. It is idempotent only **before cutover**: it also reconciles source +deletions. Never re-import over a live D1 workspace. + +Before the final copy, freeze the archived Convex workspace using its +`KLIPCODE_MIGRATED=true` environment guard. A stale browser then keeps its +unsynced changes locally until it reloads and signs in to the D1 version. +Disable the old VPS deploy timer and protect the production Worker from old +automatic deploy commands until the migration branch is published. Add the +new GitHub return URL before deploying; remove the old URL after validation. + +## Rollback + +Before D1 accepts user writes, restore the recorded pre-migration Worker +version, old GitHub callback and unset `KLIPCODE_MIGRATED` in Convex. Source data +is preserved. After D1 accepts writes, **do not switch back directly**: freeze D1 +writes, export it, reconcile new records/deletions into the source, and verify +before restoring traffic. Worker rollback alone cannot roll database state back. + +Convex code remains archived solely for the source freeze, verification and +rollback. It is not part of the deployed app backend. Remove the legacy project +and credentials only after the backup retention window and explicit approval. + +## Production cutover on 2026-09-30 + +The verified final export contains 14 users, 14 GitHub identities, 13 wrapped +keys, 43 folders and 302 snippets. App version +`6682cee2-f690-4748-a1d8-957e1fcbc996` and console version +`bf260137-d3d4-483c-8f59-934723ad4a04` completed the cutover. Production checks +confirmed GitHub sign-in resolves to an imported user, existing encrypted fields +decrypt, and encrypted create/edit/delete succeeds. The old GitHub callback was +removed and the Convex source remains frozen for recovery. + +The production Cloudflare build command currently uploads a version without +activating traffic. Once this migration is published to the production branch, +restore the deploy command to `pnpm deploy`, which applies D1 migrations before +building and deploying. Do not restore automatic activation while the branch +still contains the old Convex application. The VPS origin, tunnel and deploy +timer are disabled; infrastructure provisioning also keeps them disabled. + +The private console polls a single revision row every ten seconds while visible, +then loads cached metadata when the revision changes. It never reads ciphertext +for analytics and makes no Convex requests. + +## Legacy timestamp compatibility + +Imported records retain ISO timestamps with explicit offsets such as +`2026-07-16T11:04:39.716+00:00`, including variable fractional precision. The +backend accepts those alongside `Z` timestamps, preserves their original values, +and compares update timestamps by instant for last-write-wins. Version +`1f3b6e7d-69df-4ee1-b2a0-d08ba7bdabbf` fixes the initial validation regression +that rejected edits to these records. A real D1 regression test covers the +imported format and stale writes with a different UTC offset. diff --git a/e2e/mobile.spec.ts b/e2e/mobile.spec.ts index 698e2b7..4bec004 100644 --- a/e2e/mobile.spec.ts +++ b/e2e/mobile.spec.ts @@ -78,7 +78,9 @@ test("a swipe that falls short springs back", async ({ page }) => { test("a short flick still commits", async ({ page }) => { await gotoMobileApp(page); - await dragHorizontally(page, panel(page), { dx: -60 }); + // One native move keeps the flick fast even on a loaded CI/VPS host; eight + // separate protocol round trips can accidentally turn it into a slow drag. + await dragHorizontally(page, panel(page), { dx: -60, steps: 1 }); await expectSelected(page, "My Space"); }); diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a7b118..47b9c6a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -21,7 +21,7 @@ const eslintConfig = defineConfig([ // Isolated remote dev output (generated by KLIPCODE_DIST_DIR). ".next-remote/**", // Auto-generated by `npx convex dev`; not meant to be linted. - "convex/_generated/**", + "legacy/convex/_generated/**", ]), ]); diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..b096b4f --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,10 @@ +# Legacy source backend + +This Convex project is retained only for migration verification, the source +write freeze and rollback. No deployed application route imports it. Production +is `src/server` and `migrations`, using Workers bindings and D1. + +`KLIPCODE_MIGRATED=true` blocks legacy workspace reads and writes. Unset it only +when rolling back before D1 accepts new user writes, or after reconciling D1. +The CLI functions directory is configured in the root `convex.json` solely to +keep recovery commands repeatable. Convex packages are development dependencies. diff --git a/convex/_generated/api.d.ts b/legacy/convex/_generated/api.d.ts similarity index 100% rename from convex/_generated/api.d.ts rename to legacy/convex/_generated/api.d.ts diff --git a/convex/_generated/api.js b/legacy/convex/_generated/api.js similarity index 100% rename from convex/_generated/api.js rename to legacy/convex/_generated/api.js diff --git a/convex/_generated/dataModel.d.ts b/legacy/convex/_generated/dataModel.d.ts similarity index 100% rename from convex/_generated/dataModel.d.ts rename to legacy/convex/_generated/dataModel.d.ts diff --git a/convex/_generated/server.d.ts b/legacy/convex/_generated/server.d.ts similarity index 100% rename from convex/_generated/server.d.ts rename to legacy/convex/_generated/server.d.ts diff --git a/convex/_generated/server.js b/legacy/convex/_generated/server.js similarity index 100% rename from convex/_generated/server.js rename to legacy/convex/_generated/server.js diff --git a/convex/analytics.test.ts b/legacy/convex/analytics.test.ts similarity index 100% rename from convex/analytics.test.ts rename to legacy/convex/analytics.test.ts diff --git a/convex/analytics.ts b/legacy/convex/analytics.ts similarity index 100% rename from convex/analytics.ts rename to legacy/convex/analytics.ts diff --git a/convex/auth.config.ts b/legacy/convex/auth.config.ts similarity index 100% rename from convex/auth.config.ts rename to legacy/convex/auth.config.ts diff --git a/legacy/convex/auth.ts b/legacy/convex/auth.ts new file mode 100644 index 0000000..fae8f3a --- /dev/null +++ b/legacy/convex/auth.ts @@ -0,0 +1,37 @@ +import GitHub from "@auth/core/providers/github"; +import type { ActionCtx } from "./_generated/server"; +import { action } from "./_generated/server"; +import { v } from "convex/values"; +import { convexAuth } from "@convex-dev/auth/server"; + +// GitHub is the only provider, matching the single "Sign in with GitHub" button +// in the aside. Credentials come from the Convex deployment env +// (AUTH_GITHUB_ID / AUTH_GITHUB_SECRET), never from the client bundle. +const legacy = convexAuth({ + providers: [GitHub], +}); + +export const { auth, signOut, store, isAuthenticated } = legacy; +// Stop stale tabs from refreshing legacy tokens after the final copy. +export const signIn = action({ + args: { + provider: v.optional(v.string()), + params: v.optional(v.any()), + verifier: v.optional(v.string()), + refreshToken: v.optional(v.string()), + calledBy: v.optional(v.string()), + }, + handler: async (ctx, args) => { + if (process.env.KLIPCODE_MIGRATED === "true") + throw new Error( + "KlipCode has moved. Reload and sign in at klipcode.com.", + ); + const source = legacy.signIn as unknown as { + _handler: ( + ctx: ActionCtx, + args: Record, + ) => Promise; + }; + return source._handler(ctx, args); + }, +}); diff --git a/convex/http.ts b/legacy/convex/http.ts similarity index 74% rename from convex/http.ts rename to legacy/convex/http.ts index dfb3683..342953d 100644 --- a/convex/http.ts +++ b/legacy/convex/http.ts @@ -5,6 +5,6 @@ import { auth } from "./auth"; const http = httpRouter(); // Mounts the OAuth callback routes (/api/auth/*) on the deployment's .site domain. -auth.addHttpRoutes(http); +if (process.env.KLIPCODE_MIGRATED !== "true") auth.addHttpRoutes(http); export default http; diff --git a/convex/lib/auth.ts b/legacy/convex/lib/auth.ts similarity index 83% rename from convex/lib/auth.ts rename to legacy/convex/lib/auth.ts index 6b3e400..ed6b13c 100644 --- a/convex/lib/auth.ts +++ b/legacy/convex/lib/auth.ts @@ -13,6 +13,9 @@ type AuthCtx = { auth: { getUserIdentity: () => Promise<{ subject: string } | nu * only the first segment identifies the account. */ export async function requireUserId(ctx: AuthCtx): Promise> { + if (process.env.KLIPCODE_MIGRATED === "true") { + throw new Error("KlipCode has moved. Reload the app and sign in again; unsynced changes remain on this device."); + } const identity = await ctx.auth.getUserIdentity(); if (!identity) { diff --git a/convex/lib/hierarchy.ts b/legacy/convex/lib/hierarchy.ts similarity index 100% rename from convex/lib/hierarchy.ts rename to legacy/convex/lib/hierarchy.ts diff --git a/convex/lib/sync.ts b/legacy/convex/lib/sync.ts similarity index 100% rename from convex/lib/sync.ts rename to legacy/convex/lib/sync.ts diff --git a/convex/lib/syncCursor.ts b/legacy/convex/lib/syncCursor.ts similarity index 100% rename from convex/lib/syncCursor.ts rename to legacy/convex/lib/syncCursor.ts diff --git a/convex/lib/workspaceWrites.ts b/legacy/convex/lib/workspaceWrites.ts similarity index 100% rename from convex/lib/workspaceWrites.ts rename to legacy/convex/lib/workspaceWrites.ts diff --git a/convex/migrations.ts b/legacy/convex/migrations.ts similarity index 100% rename from convex/migrations.ts rename to legacy/convex/migrations.ts diff --git a/convex/schema.ts b/legacy/convex/schema.ts similarity index 100% rename from convex/schema.ts rename to legacy/convex/schema.ts diff --git a/convex/tsconfig.json b/legacy/convex/tsconfig.json similarity index 100% rename from convex/tsconfig.json rename to legacy/convex/tsconfig.json diff --git a/convex/userKeys.ts b/legacy/convex/userKeys.ts similarity index 100% rename from convex/userKeys.ts rename to legacy/convex/userKeys.ts diff --git a/convex/users.ts b/legacy/convex/users.ts similarity index 100% rename from convex/users.ts rename to legacy/convex/users.ts diff --git a/convex/workspace.test.ts b/legacy/convex/workspace.test.ts similarity index 100% rename from convex/workspace.test.ts rename to legacy/convex/workspace.test.ts diff --git a/convex/workspace.ts b/legacy/convex/workspace.ts similarity index 100% rename from convex/workspace.ts rename to legacy/convex/workspace.ts diff --git a/migrations/0001_d1_backend.sql b/migrations/0001_d1_backend.sql new file mode 100644 index 0000000..e8f2be0 --- /dev/null +++ b/migrations/0001_d1_backend.sql @@ -0,0 +1,168 @@ +-- Auth.js tables and KlipCode workspace. Applied explicitly with Wrangler. +CREATE TABLE IF NOT EXISTS "accounts" ( + "id" text NOT NULL, + "userId" text NOT NULL DEFAULT NULL, + "type" text NOT NULL DEFAULT NULL, + "provider" text NOT NULL DEFAULT NULL, + "providerAccountId" text NOT NULL DEFAULT NULL, + "refresh_token" text DEFAULT NULL, + "access_token" text DEFAULT NULL, + "expires_at" number DEFAULT NULL, + "token_type" text DEFAULT NULL, + "scope" text DEFAULT NULL, + "id_token" text DEFAULT NULL, + "session_state" text DEFAULT NULL, + "oauth_token_secret" text DEFAULT NULL, + "oauth_token" text DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS "sessions" ( + "id" text NOT NULL, + "sessionToken" text NOT NULL, + "userId" text NOT NULL DEFAULT NULL, + "expires" datetime NOT NULL DEFAULT NULL, + PRIMARY KEY (sessionToken) +); + +CREATE TABLE IF NOT EXISTS "users" ( + "id" text NOT NULL DEFAULT '', + "name" text DEFAULT NULL, + "email" text DEFAULT NULL, + "emailVerified" datetime DEFAULT NULL, + "image" text DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS "verification_tokens" ( + "identifier" text NOT NULL, + "token" text NOT NULL DEFAULT NULL, + "expires" datetime NOT NULL DEFAULT NULL, + PRIMARY KEY (token) +); +ALTER TABLE users ADD COLUMN created_at INTEGER NOT NULL DEFAULT 0; +ALTER TABLE users ADD COLUMN country TEXT; +CREATE UNIQUE INDEX accounts_provider_identity ON accounts(provider, providerAccountId); +CREATE INDEX accounts_user ON accounts(userId); +CREATE INDEX users_email ON users(email); +CREATE INDEX sessions_user ON sessions(userId); +CREATE TABLE user_keys (user_id TEXT PRIMARY KEY REFERENCES users(id), wrapped_dek TEXT NOT NULL); +CREATE TABLE analytics_state (id INTEGER PRIMARY KEY CHECK(id = 1), revision INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT 0); +INSERT INTO analytics_state(id) VALUES (1); +CREATE TABLE analytics_signins (id TEXT PRIMARY KEY, account TEXT NOT NULL, at INTEGER NOT NULL); +CREATE INDEX analytics_signins_account ON analytics_signins(account, at); +CREATE TABLE analytics_edits (snippet TEXT NOT NULL, account TEXT NOT NULL, language TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY(snippet, at)); +CREATE INDEX analytics_edits_account ON analytics_edits(account, at); +CREATE TABLE analytics_snippets (id TEXT PRIMARY KEY, account TEXT NOT NULL, language TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, deleted_at TEXT, pinned INTEGER NOT NULL, in_folder INTEGER NOT NULL); +CREATE INDEX analytics_snippets_account ON analytics_snippets(account); +CREATE TABLE analytics_folders (id TEXT PRIMARY KEY, account TEXT NOT NULL, created_at TEXT NOT NULL); +CREATE INDEX analytics_folders_account ON analytics_folders(account); +CREATE TABLE deletions (owner_id TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('folder','snippet')), client_id TEXT NOT NULL, deleted_at INTEGER NOT NULL, PRIMARY KEY(owner_id, kind, client_id)); +CREATE INDEX deletions_owner_time ON deletions(owner_id, deleted_at); + +CREATE TABLE folders ( + owner_id TEXT NOT NULL REFERENCES users(id), client_id TEXT NOT NULL, + analytics_id TEXT NOT NULL UNIQUE, data TEXT NOT NULL CHECK(json_valid(data)), + updated_at TEXT NOT NULL, server_updated_at INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)), + PRIMARY KEY(owner_id, client_id), CHECK(length(CAST(data AS BLOB)) <= 1900000) +); +CREATE INDEX folders_owner_time ON folders(owner_id, server_updated_at); + +CREATE TABLE snippets ( + owner_id TEXT NOT NULL REFERENCES users(id), client_id TEXT NOT NULL, + analytics_id TEXT NOT NULL UNIQUE, data TEXT NOT NULL CHECK(json_valid(data)), + updated_at TEXT NOT NULL, server_updated_at INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)), + PRIMARY KEY(owner_id, client_id), CHECK(length(CAST(data AS BLOB)) <= 1900000) +); +CREATE INDEX snippets_owner_time ON snippets(owner_id, server_updated_at); +CREATE INDEX folders_parent ON folders(owner_id, json_extract(data, '$.parentId')); +CREATE INDEX snippets_folder ON snippets(owner_id, json_extract(data, '$.folderId')); + +CREATE TRIGGER folders_cycle_insert AFTER INSERT ON folders BEGIN + SELECT RAISE(ABORT, 'Folder hierarchy cannot contain cycles') WHERE EXISTS ( + WITH RECURSIVE ancestors(client_id) AS ( + SELECT json_extract(NEW.data, '$.parentId') + UNION + SELECT json_extract(f.data, '$.parentId') FROM folders f JOIN ancestors a ON f.client_id = a.client_id WHERE f.owner_id = NEW.owner_id + ) SELECT 1 FROM ancestors WHERE client_id = NEW.client_id + ); +END; + +CREATE TRIGGER folders_cycle_update AFTER UPDATE ON folders BEGIN + SELECT RAISE(ABORT, 'Folder hierarchy cannot contain cycles') WHERE EXISTS ( + WITH RECURSIVE ancestors(client_id) AS ( + SELECT json_extract(NEW.data, '$.parentId') + UNION + SELECT json_extract(f.data, '$.parentId') FROM folders f JOIN ancestors a ON f.client_id = a.client_id WHERE f.owner_id = NEW.owner_id + ) SELECT 1 FROM ancestors WHERE client_id = NEW.client_id + ); +END; + +CREATE TRIGGER folders_analytics_insert AFTER INSERT ON folders BEGIN + INSERT INTO analytics_folders(id, account, created_at) VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data, '$.createdAt')) ON CONFLICT(id) DO UPDATE SET created_at=excluded.created_at; + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='folder' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_analytics_update AFTER UPDATE ON folders BEGIN + INSERT INTO analytics_folders(id, account, created_at) VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data, '$.createdAt')) ON CONFLICT(id) DO UPDATE SET created_at=excluded.created_at; + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='folder' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_delete AFTER DELETE ON folders BEGIN + INSERT INTO deletions(owner_id, kind, client_id, deleted_at) VALUES (OLD.owner_id,'folder',OLD.client_id,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ON CONFLICT(owner_id,kind,client_id) DO UPDATE SET deleted_at=excluded.deleted_at; + DELETE FROM analytics_folders WHERE id=OLD.analytics_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_analytics_insert AFTER INSERT ON snippets BEGIN + INSERT INTO analytics_snippets(id, account, language, created_at, updated_at, deleted_at, pinned, in_folder) +VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), json_extract(NEW.data,'$.createdAt'), json_extract(NEW.data,'$.updatedAt'), json_extract(NEW.data,'$.deletedAt'), json_extract(NEW.data,'$.isPinnedAside') OR json_extract(NEW.data,'$.isPinnedHome'), json_extract(NEW.data,'$.folderId') IS NOT NULL) +ON CONFLICT(id) DO UPDATE SET language=excluded.language, created_at=excluded.created_at, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, pinned=excluded.pinned, in_folder=excluded.in_folder; +INSERT OR IGNORE INTO analytics_edits(snippet, account, language, at) +SELECT NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), CAST(round((julianday(json_extract(NEW.data,'$.updatedAt'))-2440587.5)*86400000) AS INTEGER) +WHERE (julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.createdAt')))*86400000 > 60000 +AND (json_extract(NEW.data,'$.deletedAt') IS NULL OR abs((julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.deletedAt')))*86400000) >= 5000); + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='snippet' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_analytics_update AFTER UPDATE ON snippets BEGIN + INSERT INTO analytics_snippets(id, account, language, created_at, updated_at, deleted_at, pinned, in_folder) +VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), json_extract(NEW.data,'$.createdAt'), json_extract(NEW.data,'$.updatedAt'), json_extract(NEW.data,'$.deletedAt'), json_extract(NEW.data,'$.isPinnedAside') OR json_extract(NEW.data,'$.isPinnedHome'), json_extract(NEW.data,'$.folderId') IS NOT NULL) +ON CONFLICT(id) DO UPDATE SET language=excluded.language, created_at=excluded.created_at, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, pinned=excluded.pinned, in_folder=excluded.in_folder; +INSERT OR IGNORE INTO analytics_edits(snippet, account, language, at) +SELECT NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), CAST(round((julianday(json_extract(NEW.data,'$.updatedAt'))-2440587.5)*86400000) AS INTEGER) +WHERE (julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.createdAt')))*86400000 > 60000 +AND (json_extract(NEW.data,'$.deletedAt') IS NULL OR abs((julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.deletedAt')))*86400000) >= 5000); + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='snippet' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_delete AFTER DELETE ON snippets BEGIN + INSERT INTO deletions(owner_id, kind, client_id, deleted_at) VALUES (OLD.owner_id,'snippet',OLD.client_id,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ON CONFLICT(owner_id,kind,client_id) DO UPDATE SET deleted_at=excluded.deleted_at; + DELETE FROM analytics_snippets WHERE id=OLD.analytics_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_detach BEFORE DELETE ON folders BEGIN + UPDATE snippets SET data=json_set(data,'$.folderId',NULL), server_updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + WHERE owner_id=OLD.owner_id AND json_extract(data,'$.folderId')=OLD.client_id; +END; +CREATE TRIGGER users_created AFTER INSERT ON users BEGIN + UPDATE users SET created_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=NEW.id AND NEW.created_at=0; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; +CREATE TRIGGER sessions_created AFTER INSERT ON sessions BEGIN + INSERT OR IGNORE INTO analytics_signins(id,account,at) VALUES(NEW.id,NEW.userId,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)); + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; +CREATE TRIGGER accounts_created AFTER INSERT ON accounts BEGIN + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TABLE app_meta(key TEXT PRIMARY KEY, value INTEGER NOT NULL); +INSERT INTO app_meta VALUES('backend_epoch',CAST((julianday('now')-2440587.5)*86400000 AS INTEGER)); diff --git a/migrations/0002_signin_deduplication.sql b/migrations/0002_signin_deduplication.sql new file mode 100644 index 0000000..e06de1f --- /dev/null +++ b/migrations/0002_signin_deduplication.sql @@ -0,0 +1,3 @@ +-- Legacy snapshots and the final export can describe the same sign-in. +DELETE FROM analytics_signins WHERE rowid NOT IN (SELECT MIN(rowid) FROM analytics_signins GROUP BY account,at); +CREATE UNIQUE INDEX analytics_signins_unique_event ON analytics_signins(account,at); diff --git a/package.json b/package.json index e6a99c5..77c92a5 100644 --- a/package.json +++ b/package.json @@ -7,21 +7,23 @@ "scripts": { "build": "next build", "dev": "next dev", - "dev:backend": "convex dev", + "dev:backend": "wrangler d1 migrations apply DB --local", "start": "next start", "lint": "eslint", "test": "vitest run", "test:watch": "vitest", "test:e2e": "playwright test", "prepare": "husky", - "postinstall": "wrangler types --env-interface CloudflareEnv cloudflare-env.d.ts", + "postinstall": "pnpm cf-typegen", "preview": "opennextjs-cloudflare build && opennextjs-cloudflare preview", - "deploy": "convex deploy --cmd 'opennextjs-cloudflare build' && opennextjs-cloudflare deploy", - "upload": "convex deploy --cmd 'opennextjs-cloudflare build' && opennextjs-cloudflare upload", - "cf-typegen": "wrangler types --env-interface CloudflareEnv cloudflare-env.d.ts" + "deploy": "pnpm db:migrate && opennextjs-cloudflare build && opennextjs-cloudflare deploy", + "upload": "opennextjs-cloudflare build && opennextjs-cloudflare upload", + "cf-typegen": "wrangler types --env-file .dev.vars.example --env-interface CloudflareEnv cloudflare-env.d.ts", + "db:migrate": "wrangler d1 migrations apply DB --remote" }, "dependencies": { "@auth/core": "0.41.3", + "@auth/d1-adapter": "^1.11.3", "@codemirror/autocomplete": "^6.20.3", "@codemirror/lang-cpp": "^6.0.3", "@codemirror/lang-css": "^6.3.1", @@ -38,7 +40,6 @@ "@codemirror/language": "^6.12.3", "@codemirror/legacy-modes": "^6.5.2", "@codemirror/state": "^6.6.0", - "@convex-dev/auth": "^0.0.94", "@lezer/highlight": "^1.2.3", "@opennextjs/cloudflare": "^1.20.2", "@replit/codemirror-lang-svelte": "^6.0.0", @@ -60,7 +61,6 @@ "@uiw/codemirror-theme-vscode": "^4.25.9", "@uiw/react-codemirror": "^4.25.8", "clsx": "^2.1.1", - "convex": "^1.43.0", "dexie": "^4.3.0", "lowlight": "^3.3.0", "lucide-react": "^1.6.0", @@ -73,6 +73,7 @@ "tiptap-markdown": "^0.9.0" }, "devDependencies": { + "@convex-dev/auth": "^0.0.94", "@edge-runtime/vm": "^5.0.0", "@playwright/test": "^1.61.1", "@tailwindcss/postcss": "^4", @@ -81,12 +82,14 @@ "@types/react": "^19", "@types/react-dom": "^19", "babel-plugin-react-compiler": "1.0.0", + "convex": "^1.43.0", "convex-test": "^0.0.54", "eslint": "^9", "eslint-config-next": "16.2.11", "fake-indexeddb": "^6.2.5", "husky": "^9.1.7", "jsdom": "^30.0.1", + "miniflare": "4.20260708.1", "tailwindcss": "^4", "typescript": "^5", "vite": "^8.2.0", diff --git a/playwright.config.ts b/playwright.config.ts index a68d1c8..451a762 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,11 +5,13 @@ import { defineConfig, devices } from "@playwright/test"; * * Cloud-sync behaviour is covered where it actually lives rather than through * the browser: `src/__tests__/sync.test.ts` drives the client sync engine - * against an in-memory deployment, and `convex/workspace.test.ts` runs the real + * against an in-memory deployment, and `src/server/workspace.test.ts` runs the real * backend functions (ownership, last-write-wins, cycle rejection, delete - * cascade) on Convex's test harness. Both run in CI with no Docker and no live + * cascade) on local D1. Both run in CI with no Docker and no live * deployment — which the previous Supabase-backed sync suite could not do. */ +const testPort = Number(process.env.KLIPCODE_TEST_PORT ?? 3000); + export default defineConfig({ testDir: "e2e", fullyParallel: true, @@ -17,7 +19,7 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, reporter: process.env.CI ? "github" : "list", use: { - baseURL: "http://localhost:3000", + baseURL: `http://127.0.0.1:${testPort}`, trace: "on-first-retry", }, projects: [ @@ -34,8 +36,8 @@ export default defineConfig({ ], webServer: [ { - command: "pnpm dev", - url: "http://localhost:3000", + command: `pnpm exec next dev --hostname 127.0.0.1 --port ${testPort}`, + url: `http://127.0.0.1:${testPort}`, reuseExistingServer: !process.env.CI, timeout: 120_000, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2f4d80..791713c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@auth/core': specifier: 0.41.3 version: 0.41.3 + '@auth/d1-adapter': + specifier: ^1.11.3 + version: 1.11.3 '@codemirror/autocomplete': specifier: ^6.20.3 version: 6.20.3 @@ -59,9 +62,6 @@ importers: '@codemirror/state': specifier: ^6.6.0 version: 6.6.0 - '@convex-dev/auth': - specifier: ^0.0.94 - version: 0.0.94(@auth/core@0.41.3)(convex@1.43.0(react@19.2.4))(react@19.2.4) '@lezer/highlight': specifier: ^1.2.3 version: 1.2.3 @@ -125,9 +125,6 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 - convex: - specifier: ^1.43.0 - version: 1.43.0(react@19.2.4) dexie: specifier: ^4.3.0 version: 4.3.0 @@ -159,6 +156,9 @@ importers: specifier: ^0.9.0 version: 0.9.0(@tiptap/core@2.27.2(@tiptap/pm@2.27.2)) devDependencies: + '@convex-dev/auth': + specifier: ^0.0.94 + version: 0.0.94(@auth/core@0.41.3)(convex@1.43.0(react@19.2.4))(react@19.2.4) '@edge-runtime/vm': specifier: ^5.0.0 version: 5.0.0 @@ -183,6 +183,9 @@ importers: babel-plugin-react-compiler: specifier: 1.0.0 version: 1.0.0 + convex: + specifier: ^1.43.0 + version: 1.43.0(react@19.2.4) convex-test: specifier: ^0.0.54 version: 0.0.54(convex@1.43.0(react@19.2.4)) @@ -201,6 +204,9 @@ importers: jsdom: specifier: ^30.0.1 version: 30.0.1(@noble/hashes@1.8.0) + miniflare: + specifier: 4.20260708.1 + version: 4.20260708.1 tailwindcss: specifier: ^4 version: 4.2.2 @@ -303,6 +309,9 @@ packages: nodemailer: optional: true + '@auth/d1-adapter@1.11.3': + resolution: {integrity: sha512-F71eQVTJzP0igCaPMFqE7EUp62sea6COGQs+6rF7kN1998KBRKy+I0xennOz00gnL/dMOM82PP0GvE5Vtpjm7g==} + '@aws-crypto/sha1-browser@5.2.0': resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} @@ -5088,6 +5097,14 @@ snapshots: preact: 10.24.3 preact-render-to-string: 6.5.11(preact@10.24.3) + '@auth/d1-adapter@1.11.3': + dependencies: + '@auth/core': 0.41.3 + transitivePeerDependencies: + - '@simplewebauthn/browser' + - '@simplewebauthn/server' + - nodemailer + '@aws-crypto/sha1-browser@5.2.0': dependencies: '@aws-crypto/supports-web-crypto': 5.2.0 diff --git a/public/sw.js b/public/sw.js index 744eec4..2c3007e 100644 --- a/public/sw.js +++ b/public/sw.js @@ -13,7 +13,7 @@ * * Bump CACHE_VERSION to invalidate previously cached responses. */ -const CACHE_VERSION = "v1"; +const CACHE_VERSION = "v2"; const CACHE_NAME = `klipcode-${CACHE_VERSION}`; const APP_SHELL = "/app"; diff --git a/scripts/migrate-convex-to-d1.py b/scripts/migrate-convex-to-d1.py new file mode 100644 index 0000000..7b3b2c0 --- /dev/null +++ b/scripts/migrate-convex-to-d1.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Generate an idempotent D1 import from a protected Convex export. Never decrypts. + +Run only before the D1 cutover, with Convex workspace writes frozen. SQL files +contain private data: keep them outside the repository, with mode 0600. +""" +import argparse +import hashlib +import json +import os +import subprocess +import urllib.request +import urllib.error +import tomllib +import zipfile +from pathlib import Path +from datetime import datetime, timezone + +parser = argparse.ArgumentParser() +parser.add_argument('export', type=Path) +parser.add_argument('--sql', type=Path) +parser.add_argument('--analytics', type=Path) +parser.add_argument('--verify', action='store_true') +parser.add_argument('--local', action='store_true') +parser.add_argument('--apply-remote', action='store_true', help='Prepared REST batches; only before cutover') +parser.add_argument('--source-frozen', action='store_true', help='Confirm the source write guard is enabled') +parser.add_argument('--account-id') +parser.add_argument('--database-id') +parser.add_argument('--oauth-config', type=Path, help='Wrangler OAuth config; otherwise use CLOUDFLARE_API_TOKEN') +args = parser.parse_args() +archive = zipfile.ZipFile(args.export) +def table(name): + try: return [json.loads(line) for line in archive.read(name+'/documents.jsonl').splitlines() if line] + except KeyError: + raise ValueError(f'Missing {name}/documents.jsonl in export; refusing to reconcile data') from None +def quote(value): + if value is None: return 'NULL' + if isinstance(value, (int,float)): return str(round(value)) + return "'"+str(value).replace("'","''")+"'" +def iso(ms): return datetime.fromtimestamp(ms/1000, timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z') +def hashid(s): return hashlib.sha256(s.encode()).hexdigest()[:8] +users=table('users'); keys=table('userKeys'); folders=table('folders'); snippets=table('snippets') +clock=round(datetime.now(timezone.utc).timestamp()*1000) +statements=[] +prepared=[] +def insert(name, fields, values, suffix=''): + statements.append(f"INSERT INTO {name}({','.join(fields)}) VALUES({','.join(quote(x) for x in values)}) {suffix};") + prepared.append((name, {'sql':f"INSERT INTO {name}({','.join(fields)}) VALUES({','.join('?' for _ in values)}) {suffix}", 'params':values})) +for u in users: + insert('users',['id','name','email','image','emailVerified','created_at','country'],[u['_id'],u.get('name'),u.get('email'),u.get('image'),iso(u['emailVerificationTime']) if u.get('emailVerificationTime') else None,round(u['_creationTime']),u.get('country')], 'ON CONFLICT(id) DO UPDATE SET name=excluded.name,email=excluded.email,image=excluded.image,emailVerified=excluded.emailVerified') +for a in table('authAccounts'): + if a['provider']!='github': raise ValueError('Unsupported legacy provider') + insert('accounts',['id','userId','type','provider','providerAccountId'],[a['_id'],a['userId'],'oauth',a['provider'],a['providerAccountId']], 'ON CONFLICT(provider,providerAccountId) DO NOTHING') +for k in keys: + insert('user_keys',['user_id','wrapped_dek'],[k['userId'],k['wrappedDek']], 'ON CONFLICT(user_id) DO NOTHING') +common=['clientId','createdAt','updatedAt','deletedAt','cryptoVersion','isPinnedAside','isPinnedHome'] +def wire(row,kind): + return {k:row[k] for k in common+(['name','parentId'] if kind=='folders' else ['title','code','language','folderId'])} +for name,records in [('folders',folders),('snippets',snippets)]: + for r in records: + insert(name,['owner_id','client_id','analytics_id','data','updated_at','server_updated_at'],[r['ownerId'],r['clientId'],r['_id'],json.dumps(wire(r,name),separators=(',',':'),ensure_ascii=True),r['updatedAt'],clock], 'ON CONFLICT(owner_id,client_id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at,server_updated_at=excluded.server_updated_at WHERE excluded.updated_at>='+name+'.updated_at AND excluded.data<>'+name+'.data') + # A second pre-cutover import reconciles deletions made since the first copy. + present=json.dumps([r['_id'] for r in records]) + statements.append(f'DELETE FROM {name} WHERE analytics_id NOT IN (SELECT value FROM json_each({quote(present)}));') + prepared.append((name, {'sql':f'DELETE FROM {name} WHERE analytics_id NOT IN (SELECT value FROM json_each(?))', 'params':[present]})) +for s in table('authSessions'): + insert('analytics_signins',['id','account','at'],[s['_id'],s['userId'],round(s['_creationTime'])], 'ON CONFLICT DO NOTHING') +if args.analytics: + previous=json.loads(args.analytics.read_text()) + owners={hashid(u['_id']):u['_id'] for u in users} + snippet_ids={hashid(s['_id']):s['_id'] for s in snippets} + for e in previous.get('edits',[]): + if e['account'] in owners: + insert('analytics_edits',['snippet','account','language','at'],[snippet_ids.get(e['snippet'],'anon:'+e['snippet']),owners[e['account']],e['language'],e['at']], 'ON CONFLICT(snippet,at) DO NOTHING') + for s in previous.get('sessions',[]): + if s['account'] in owners: + insert('analytics_signins',['id','account','at'],['legacy:'+s['account']+':'+str(s['at']),owners[s['account']],s['at']], 'ON CONFLICT DO NOTHING') +# Only the final copy's epoch is trusted; old device cursors force a full pull. +statements.append(f"UPDATE app_meta SET value={clock} WHERE key='backend_epoch';") +prepared.append(('app_meta', {'sql':"UPDATE app_meta SET value=? WHERE key='backend_epoch'", 'params':[clock]})) +if args.sql: + fd=os.open(args.sql,os.O_WRONLY|os.O_CREAT|os.O_TRUNC,0o600) + with os.fdopen(fd,'w') as f: f.write('\n'.join(statements)+'\n') + print(json.dumps({'users':len(users),'keys':len(keys),'folders':len(folders),'snippets':len(snippets),'statements':len(statements),'sql_bytes':args.sql.stat().st_size})) +if args.apply_remote: + if args.local or not args.source_frozen or not args.account_id or not args.database_id: + parser.error('--apply-remote requires --source-frozen, --account-id and --database-id, without --local') + token=os.environ.get('CLOUDFLARE_API_TOKEN') + if not token and args.oauth_config: + token=tomllib.loads(args.oauth_config.read_text()).get('oauth_token') + if not token: + parser.error('Provide CLOUDFLARE_API_TOKEN or --oauth-config; never pass tokens on the command line') + endpoint=f'https://api.cloudflare.com/client/v4/accounts/{args.account_id}/d1/database/{args.database_id}/query' + def remote(body): + request=urllib.request.Request(endpoint, data=json.dumps(body).encode(), headers={'Authorization':'Bearer '+token,'Content-Type':'application/json'}) + try: + with urllib.request.urlopen(request,timeout=60) as response: + result=json.load(response) + except urllib.error.HTTPError as error: + # API errors may echo private SQL parameters; report only the code. + raise RuntimeError(f'Cloudflare HTTP {error.code}; import stopped, do not activate traffic') from None + if not result.get('success') or any(not r.get('success') for r in result.get('result',[])): + raise RuntimeError('D1 import failed; do not activate traffic') + return result['result'] + if remote({'sql':'SELECT count(*) AS count FROM sessions'})[0]['results'][0]['count']: + raise RuntimeError('D1 already has login sessions. Refusing to import over a live workspace.') + stored={} + for kind in ['folders','snippets']: + stored[kind]={(r['owner_id'],r['client_id']):r for r in remote({'sql':f'SELECT owner_id,client_id,analytics_id,data FROM {kind}'})[0]['results']} + pending=[] + skipped=0 + for kind, statement in prepared: + if kind in stored and statement['sql'].startswith('INSERT'): + owner,client,analytics_id,data,*_=statement['params'] + old=stored[kind].get((owner,client)) + if old and old['analytics_id']==analytics_id and json.loads(old['data'])==json.loads(data): + skipped+=1 + continue + pending.append(statement) + # Large SQL imports can fail with D1_RESET_DO. Bind values instead, skipping + # unchanged ciphertext and bounding each request. The source stays frozen; + # verification must pass before any version accepts D1 traffic. + batch=[] + total=0 + size=0 + for statement in pending: + length=len(json.dumps(statement).encode()) + if batch and (len(batch)>=100 or size+length>2000000): + remote({'batch':batch}) + total+=len(batch) + batch=[] + size=0 + batch.append(statement) + size+=length + if batch: + remote({'batch':batch}) + total+=len(batch) + print(json.dumps({'applied_statements':total,'unchanged_workspace_records_skipped':skipped})) +if args.verify: + def query(sql): + proc=subprocess.run(['pnpm','exec','wrangler','d1','execute','DB','--local' if args.local else '--remote','--json','--command',sql],capture_output=True,text=True,check=True) + return json.loads(proc.stdout)[0]['results'] + stored_users=query('SELECT id FROM users'); assert {u['id'] for u in stored_users}=={u['_id'] for u in users},'User mismatch' + stored_keys=query('SELECT user_id,wrapped_dek FROM user_keys');assert {k['user_id']:k['wrapped_dek'] for k in stored_keys}=={k['userId']:k['wrappedDek'] for k in keys},'Key mismatch' + for kind, records in [('folders',folders),('snippets',snippets)]: + stored=query(f'SELECT owner_id,client_id,analytics_id,data FROM {kind}') + expected={(r['ownerId'],r['clientId']):(r['_id'],wire(r,kind)) for r in records} + actual={(r['owner_id'],r['client_id']):(r['analytics_id'],json.loads(r['data'])) for r in stored} + assert actual==expected,kind+' mismatch' + accounts=query('SELECT userId,provider,providerAccountId FROM accounts') + assert {(a['userId'],a['provider'],a['providerAccountId']) for a in accounts}=={(a['userId'],a['provider'],a['providerAccountId']) for a in table('authAccounts')},'GitHub account mismatch' + print('Verified all account ids, GitHub identities, ciphertext fields and wrapped keys byte for byte.') diff --git a/scripts/test-migrate-convex-to-d1.py b/scripts/test-migrate-convex-to-d1.py new file mode 100644 index 0000000..92de62f --- /dev/null +++ b/scripts/test-migrate-convex-to-d1.py @@ -0,0 +1,59 @@ +import io, json, runpy, sqlite3, tempfile, unittest, zipfile +from pathlib import Path +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[1] +class PreparedImportTest(unittest.TestCase): + def setUp(self): + self.db=sqlite3.connect(':memory:'); self.db.row_factory=sqlite3.Row + for p in sorted((ROOT/'migrations').glob('*.sql')): self.db.executescript(p.read_text()) + self.base={'clientId':'c1','createdAt':'2026-09-01T00:00:00.000Z','updatedAt':'2026-09-01T00:00:00.000Z','deletedAt':None,'cryptoVersion':1,'isPinnedAside':False,'isPinnedHome':False} + self.data={'users':[{'_id':'u1','_creationTime':1790000000000,'email':'test@example.com'}],'authAccounts':[{'_id':'a1','userId':'u1','provider':'github','providerAccountId':'42'}],'userKeys':[{'userId':'u1','wrappedDek':'wrapped-test'}],'folders':[],'snippets':[{**self.base,'_id':'s1','ownerId':'u1','title':'ciphertext-test','code':'large-ciphertext-'*10000,'language':'plaintext','folderId':None}],'authSessions':[]} + self.writes=0; self.largest_sql=0 + def api(self,request,timeout): + body=json.loads(request.data); queries=body.get('batch',[body]); results=[] + with self.db: + for q in queries: + self.largest_sql=max(self.largest_sql,len(q['sql'])) + if not q['sql'].startswith('SELECT'): self.writes+=1 + c=self.db.execute(q['sql'],q.get('params',[])) + results.append({'success':True,'results':[dict(r) for r in c.fetchall()] if c.description else []}) + return io.BytesIO(json.dumps({'success':True,'result':results}).encode()) + def run_import(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'source.zip' + with zipfile.ZipFile(p,'w') as z: + for name,rows in self.data.items(): z.writestr(name+'/documents.jsonl','\n'.join(json.dumps(r) for r in rows)) + args=['importer',str(p),'--apply-remote','--source-frozen','--account-id','test-account','--database-id','test-db'] + with patch('sys.argv',args),patch.dict('os.environ',{'CLOUDFLARE_API_TOKEN':'test'}),patch('urllib.request.urlopen',self.api): + runpy.run_path(str(ROOT/'scripts/migrate-convex-to-d1.py'),run_name='__main__') + def test_large_ciphertext_bound_and_unchanged_edits_not_duplicated(self): + self.run_import() + self.assertEqual(json.loads(self.db.execute('SELECT data FROM snippets').fetchone()[0])['code'],self.data['snippets'][0]['code']) + revision=self.db.execute('SELECT revision FROM analytics_state').fetchone()[0] + self.run_import() + self.assertEqual(self.db.execute('SELECT revision FROM analytics_state').fetchone()[0],revision) + self.assertLess(self.largest_sql,1000) + def test_final_copy_reconciles_edits_and_deletions(self): + self.run_import() + self.data['snippets'][0]['updatedAt']='2026-09-02T00:00:00.000Z';self.data['snippets'][0]['code']='edited-ciphertext' + self.run_import() + self.assertEqual(json.loads(self.db.execute('SELECT data FROM snippets').fetchone()[0])['code'],'edited-ciphertext') + self.data['snippets']=[]; self.run_import() + self.assertEqual(self.db.execute('SELECT count(*) FROM snippets').fetchone()[0],0) + self.assertEqual(self.db.execute('SELECT count(*) FROM analytics_snippets').fetchone()[0],0) + def test_live_login_guard_runs_before_any_mutation(self): + self.db.execute("INSERT INTO sessions(id,sessionToken,userId,expires) VALUES('session','hash','u1','2026-10-01T00:00:00.000Z')") + with self.assertRaisesRegex(RuntimeError,'login sessions'): self.run_import() + self.assertEqual(self.writes,0) + def test_missing_required_table_stops_before_any_mutation(self): + for name in list(self.data): + with self.subTest(table=name): + rows=self.data.pop(name) + try: + with self.assertRaisesRegex(ValueError,'Missing '+name+'/documents.jsonl'): + self.run_import() + self.assertEqual(self.writes,0) + finally: + self.data[name]=rows +if __name__=='__main__': unittest.main() diff --git a/src/__tests__/seed.test.ts b/src/__tests__/seed.test.ts index 0995393..44ada5c 100644 --- a/src/__tests__/seed.test.ts +++ b/src/__tests__/seed.test.ts @@ -19,10 +19,10 @@ function makeClient() { }; } -vi.mock("@/lib/convex", () => ({ - isConvexConfigured: () => mockClient !== null, - getConvexBrowserClient: () => mockClient, -})); +vi.mock("@/lib/cloud", () => ({cloud:{hasContent: async()=> { + if (!mockClient || !mockSignedIn) throw new Error("Not authenticated"); + return mockCloud.folders>0 || mockCloud.snippets>0; +}}})); import { db } from "@/lib/db"; import { readInitialWorkspace, seedWelcomeContent } from "@/lib/seed"; diff --git a/src/__tests__/sync.test.ts b/src/__tests__/sync.test.ts index b6210ee..0c61458 100644 --- a/src/__tests__/sync.test.ts +++ b/src/__tests__/sync.test.ts @@ -1,6 +1,5 @@ // fake-indexeddb/auto is loaded via vitest setupFiles (vitest.config.ts). import { describe, it, expect, beforeEach, vi } from "vitest"; -import { getFunctionName, type FunctionReference } from "convex/server"; import { assertNoFolderCycles, collectDescendantFolderIds } from "@convex/lib/hierarchy"; import type { CloudFolder, CloudSnippet, FolderRecord, SnippetRecord } from "@/lib/types"; @@ -147,36 +146,13 @@ function changes({ since }: { since: number | null }) { }; } -const convexClient = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async query(reference: FunctionReference<"query">, args?: never): Promise { - switch (getFunctionName(reference)) { - case "workspace:changes": - return changes(args!); - case "workspace:list": - return { folders: ownedFolders().map(stripOwner), snippets: ownedSnippets().map(stripOwner) }; - case "workspace:hasContent": - return ownedFolders().length > 0 || ownedSnippets().length > 0; - default: - throw new Error(`Unexpected query ${getFunctionName(reference)}`); - } - }, - async mutation(reference: FunctionReference<"mutation">, args: never) { - switch (getFunctionName(reference)) { - case "workspace:push": - return push(args); - case "workspace:remove": - return remove(args); - default: - throw new Error(`Unexpected mutation ${getFunctionName(reference)}`); - } - }, +const cloudClient = { + changes: async (since: number | null) => changes({since}), + push: async (body: {folders: CloudFolder[]; snippets:CloudSnippet[]}) => push(body), + remove: async (body: {folderIds:string[];snippetIds:string[]}) => remove(body), + hasContent: async () => ownedFolders().length>0 || ownedSnippets().length>0, }; - -vi.mock("@/lib/convex", () => ({ - isConvexConfigured: () => true, - getConvexBrowserClient: () => convexClient, -})); +vi.mock("@/lib/cloud", () => ({cloud:{changes:(since:number|null)=>cloudClient.changes(since),push:(body:{folders:CloudFolder[];snippets:CloudSnippet[]})=>cloudClient.push(body),remove:(body:{folderIds:string[];snippetIds:string[]})=>cloudClient.remove(body),hasContent:()=>cloudClient.hasContent()}})); // ── Mocked encryption key ────────────────────────────────────────────────────── // `null` (the default) runs sync in plaintext mode, matching the pre-encryption @@ -443,6 +419,16 @@ describe("syncDirtyWorkspace() batching", () => { expect(order.indexOf(child.id)).toBeLessThan(order.indexOf(grandchild.id)); }); + it("bounds the combined folder/snippet payload, not just each list", async () => { + await db.folders.bulkAdd([makeFolder({dirty:true,name:'f'.repeat(1200000)}),makeFolder({dirty:true,name:'g'.repeat(1200000)})]); + await db.snippets.bulkAdd([makeSnippet({dirty:true,code:'a'.repeat(1200000)}),makeSnippet({dirty:true,code:'b'.repeat(1200000)})]); + const calls=vi.spyOn(cloudClient,'push'); + await syncDirtyWorkspace(USER); + expect(calls).toHaveBeenCalledTimes(2); + for (const [body] of calls.mock.calls) expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(4000000); + calls.mockRestore(); + }); + it("fails clearly instead of sending an oversized record forever", async () => { const oversized = makeSnippet({ dirty: true, code: "x".repeat(4_100_000) }); await db.snippets.add(oversized); @@ -822,26 +808,8 @@ describe("fetchCloudWorkspace() incremental pull", () => { expect((await db.snippets.get(snippet.id))?.title).toBe("back"); }); - it("falls back to the full list when the backend has no incremental query yet", async () => { - const snippet = makeSnippet(); - pushFromElsewhere([], [cloudSnippet(snippet)]); - await db.syncCursors.put({ userId: USER, cursor: 1 }); - backendHasChanges = false; - - await fetchCloudWorkspace(USER); - - expect(await db.snippets.get(snippet.id)).toBeDefined(); - // Writes on that backend are not stamped, so the cursor must not be trusted. - expect(await db.syncCursors.get(USER)).toBeUndefined(); - - // Deletions still propagate, by absence, exactly as before. - cloud.snippets = []; - await fetchCloudWorkspace(USER); - expect(await db.snippets.get(snippet.id)).toBeUndefined(); - }); - it("does not turn other failures into a second, full read", async () => { - const failing = vi.spyOn(convexClient, "query").mockRejectedValueOnce(new Error("Not authenticated")); + const failing = vi.spyOn(cloudClient, "changes").mockRejectedValueOnce(new Error("Not authenticated")); await expect(fetchCloudWorkspace(USER)).rejects.toThrow("Not authenticated"); expect(failing).toHaveBeenCalledTimes(1); diff --git a/src/app/[locale]/layout.tsx b/src/app/[locale]/layout.tsx index e124c14..35bd212 100644 --- a/src/app/[locale]/layout.tsx +++ b/src/app/[locale]/layout.tsx @@ -1,5 +1,5 @@ import { Geist, Geist_Mono } from "next/font/google"; -import { ConvexClientProvider } from "@/components/ConvexClientProvider"; +import { CloudClientProvider } from "@/components/CloudClientProvider"; import { ServiceWorkerRegistration } from "@/components/ServiceWorkerRegistration"; import { ASIDE_WIDTH_INIT_SCRIPT } from "@/lib/asideWidth"; import { THEME_INIT_SCRIPT } from "@/lib/theme"; @@ -48,7 +48,7 @@ export default async function LocaleLayout({ {/* Same idea for the aside's stored width: applied before the panel paints so it never animates out from the default on every load. */}