From d14799c393b1ef214305e1eb8fd76b7e4f661b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20Mart=C3=ADnez?= Date: Thu, 1 Oct 2026 09:50:14 +0200 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9C=A8=20feat:=20migrate=20KlipCode=20ba?= =?UTF-8?q?ckend=20and=20GitHub=20sessions=20to=20D1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Preserve account ids, wrapped keys and ciphertext while moving workspace sync and authentication to Cloudflare. Keep the frozen Convex source for verified recovery, use indexed deltas and atomic batches, and fail closed on encryption errors. Accept legacy ISO offsets to prevent imported records from failing subsequent saves. Cover ownership, stale writes, cascades, encryption boundaries and migration safety in regression tests. --- .dev.vars.example | 15 +- .env.example | 21 +- .env.production | 23 +- .github/workflows/ci.yml | 3 + README.md | 38 +- convex.json | 3 + convex/auth.ts | 9 - docs/audit/engineering-audit.md | 3 + docs/d1-migration.md | 110 ++++++ e2e/mobile.spec.ts | 4 +- eslint.config.mjs | 2 +- legacy/README.md | 10 + {convex => legacy/convex}/_generated/api.d.ts | 0 {convex => legacy/convex}/_generated/api.js | 0 .../convex}/_generated/dataModel.d.ts | 0 .../convex}/_generated/server.d.ts | 0 .../convex}/_generated/server.js | 0 {convex => legacy/convex}/analytics.test.ts | 0 {convex => legacy/convex}/analytics.ts | 0 {convex => legacy/convex}/auth.config.ts | 0 legacy/convex/auth.ts | 37 ++ {convex => legacy/convex}/http.ts | 2 +- {convex => legacy/convex}/lib/auth.ts | 3 + {convex => legacy/convex}/lib/hierarchy.ts | 0 {convex => legacy/convex}/lib/sync.ts | 0 {convex => legacy/convex}/lib/syncCursor.ts | 0 .../convex}/lib/workspaceWrites.ts | 0 {convex => legacy/convex}/migrations.ts | 0 {convex => legacy/convex}/schema.ts | 0 {convex => legacy/convex}/tsconfig.json | 0 {convex => legacy/convex}/userKeys.ts | 0 {convex => legacy/convex}/users.ts | 0 {convex => legacy/convex}/workspace.test.ts | 0 {convex => legacy/convex}/workspace.ts | 0 migrations/0001_d1_backend.sql | 168 ++++++++ migrations/0002_signin_deduplication.sql | 3 + package.json | 17 +- playwright.config.ts | 12 +- pnpm-lock.yaml | 29 +- public/sw.js | 2 +- scripts/migrate-convex-to-d1.py | 151 ++++++++ scripts/test-migrate-convex-to-d1.py | 49 +++ src/__tests__/seed.test.ts | 8 +- src/__tests__/sync.test.ts | 66 +--- src/app/[locale]/layout.tsx | 4 +- src/app/api/auth/[...auth]/route.ts | 3 + src/app/api/crypto/dek/route.ts | 130 ++----- src/app/api/generate-title/route.ts | 2 +- src/app/api/images/route.ts | 2 +- src/app/api/workspace/[operation]/route.ts | 90 +++++ src/components/CloudClientProvider.tsx | 129 +++++++ src/components/ConvexClientProvider.tsx | 70 ---- src/components/MarkdownPreview/imageApi.ts | 9 - src/hooks/useCloudSession.ts | 11 +- src/hooks/useWorkspaceMutations.ts | 6 +- src/lib/authToken.ts | 24 -- src/lib/cloud.ts | 49 +++ src/lib/convex.ts | 24 -- src/lib/convexEnv.ts | 19 - src/lib/convexServer.ts | 66 ---- src/lib/encryptionKey.ts | 33 +- src/lib/seed.ts | 13 +- src/lib/sync.ts | 358 +++++++++++------- src/lib/types.ts | 2 +- src/server/auth.ts | 173 +++++++++ src/server/env.ts | 7 + src/server/workspace.test.ts | 286 ++++++++++++++ src/server/workspace.ts | 232 ++++++++++++ tsconfig.json | 2 +- vitest.config.mts | 2 +- wrangler.jsonc | 32 +- 71 files changed, 1898 insertions(+), 668 deletions(-) create mode 100644 convex.json delete mode 100644 convex/auth.ts create mode 100644 docs/d1-migration.md create mode 100644 legacy/README.md rename {convex => legacy/convex}/_generated/api.d.ts (100%) rename {convex => legacy/convex}/_generated/api.js (100%) rename {convex => legacy/convex}/_generated/dataModel.d.ts (100%) rename {convex => legacy/convex}/_generated/server.d.ts (100%) rename {convex => legacy/convex}/_generated/server.js (100%) rename {convex => legacy/convex}/analytics.test.ts (100%) rename {convex => legacy/convex}/analytics.ts (100%) rename {convex => legacy/convex}/auth.config.ts (100%) create mode 100644 legacy/convex/auth.ts rename {convex => legacy/convex}/http.ts (74%) rename {convex => legacy/convex}/lib/auth.ts (83%) rename {convex => legacy/convex}/lib/hierarchy.ts (100%) rename {convex => legacy/convex}/lib/sync.ts (100%) rename {convex => legacy/convex}/lib/syncCursor.ts (100%) rename {convex => legacy/convex}/lib/workspaceWrites.ts (100%) rename {convex => legacy/convex}/migrations.ts (100%) rename {convex => legacy/convex}/schema.ts (100%) rename {convex => legacy/convex}/tsconfig.json (100%) rename {convex => legacy/convex}/userKeys.ts (100%) rename {convex => legacy/convex}/users.ts (100%) rename {convex => legacy/convex}/workspace.test.ts (100%) rename {convex => legacy/convex}/workspace.ts (100%) create mode 100644 migrations/0001_d1_backend.sql create mode 100644 migrations/0002_signin_deduplication.sql create mode 100644 scripts/migrate-convex-to-d1.py create mode 100644 scripts/test-migrate-convex-to-d1.py create mode 100644 src/app/api/auth/[...auth]/route.ts create mode 100644 src/app/api/workspace/[operation]/route.ts create mode 100644 src/components/CloudClientProvider.tsx delete mode 100644 src/components/ConvexClientProvider.tsx delete mode 100644 src/lib/authToken.ts create mode 100644 src/lib/cloud.ts delete mode 100644 src/lib/convex.ts delete mode 100644 src/lib/convexEnv.ts delete mode 100644 src/lib/convexServer.ts create mode 100644 src/server/auth.ts create mode 100644 src/server/env.ts create mode 100644 src/server/workspace.test.ts create mode 100644 src/server/workspace.ts diff --git a/.dev.vars.example b/.dev.vars.example index efdcebb..7c37ed9 100644 --- a/.dev.vars.example +++ b/.dev.vars.example @@ -1,10 +1,7 @@ -# Wrangler does not read .env, so `pnpm preview` needs the same values here. -# Copy to .dev.vars and fill in. -# -# The Convex deployment the previewed build talks to. `pnpm dev:backend` puts -# this in .env.local; copy the same value here. -NEXT_PUBLIC_CONVEX_URL= - -# See .env.example — same key, same warning: back it up, and never set it as a -# Convex environment variable. +# Copy to .dev.vars. Never commit real credentials or the existing master key. +AUTH_URL=http://localhost:3000 +AUTH_SECRET= +AUTH_GITHUB_ID= +AUTH_GITHUB_SECRET= +# Base64-encoded 32-byte key. Production must keep its existing value forever. ENCRYPTION_MASTER_KEY= diff --git a/.env.example b/.env.example index 05203a1..dd0a4e7 100644 --- a/.env.example +++ b/.env.example @@ -1,18 +1,3 @@ -# Copy to .env for local development with `pnpm dev` (Node runtime). -# For `pnpm preview` / `pnpm deploy` (Cloudflare Workers runtime), the same -# values must live in .dev.vars instead — wrangler does not read .env. -# In production, set them as Worker secrets: wrangler secret put -# -# NEXT_PUBLIC_CONVEX_URL is NOT listed here: `pnpm dev:backend` writes it (and -# CONVEX_DEPLOYMENT) into .env.local for you. Without a deployment URL, sign-in -# and cloud storage are unavailable; only the device-bound guest workspace is -# available. - -# Master key (KEK) that wraps every per-user data-encryption key. It must be -# 32 random bytes in base64 — generate one with: openssl rand -base64 32 -# Deliberately kept on Cloudflare and NEVER set as a Convex environment -# variable: that separation is what stops the encrypted records and the key -# that opens them from living with the same provider. -# Losing it makes all encrypted cloud records unrecoverable: back it up in a -# password manager. If it is unset, the app syncs in plaintext (cryptoVersion 0). -ENCRYPTION_MASTER_KEY= +# Runtime configuration belongs in .dev.vars; see .dev.vars.example. +# Keep remote Images/AI disabled when testing without Cloudflare credentials. +KLIPCODE_REMOTE_BINDINGS=false diff --git a/.env.production b/.env.production index 7a3478f..b2bd9d5 100644 --- a/.env.production +++ b/.env.production @@ -1,21 +1,2 @@ -# Production build values. Committed on purpose. -# -# NEXT_PUBLIC_* is inlined into the client bundle at BUILD time, so it has to be -# present when the build runs — setting it as a Worker runtime variable does -# nothing, the bundle is already compiled by then. Cloudflare Workers Builds -# runs `opennextjs-cloudflare build` on its own runners, where neither -# .env.local nor the `convex deploy --cmd` wrapper in `pnpm deploy` exists, so -# without this file the production build compiles with no deployment URL and the -# app silently falls back to a device-only guest workspace. -# -# Keeping it here rather than in the Cloudflare dashboard means the value is -# reviewable, versioned, and cannot drift out of sync with the deployment. -# -# This is not a secret: the URL ships inside the client bundle either way, and -# access is gated by the identity checks inside each Convex function, never by -# the URL being unknown. The actual secret (ENCRYPTION_MASTER_KEY) stays a -# Cloudflare Worker secret and is never committed. -# -# .env.local wins over this file, so local builds keep pointing at the dev -# deployment; only builds without a .env.local (i.e. CI/Cloudflare) use it. -NEXT_PUBLIC_CONVEX_URL=https://befitting-badger-749.convex.cloud +# The cloud backend is same-origin and configured through runtime D1 bindings. +# No deployment URL or credentials are embedded in the browser bundle. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df4e9f5..f6ab409 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,9 @@ jobs: - name: Test run: pnpm test + - name: Test migration importer + run: python3 scripts/test-migrate-convex-to-d1.py + - name: Build run: pnpm build diff --git a/README.md b/README.md index 08bb2e4..8298b49 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ guest workspace that stays on the device you are using. - Write and preview Markdown documents in a rich TipTap editor. - Generate a name for an untitled snippet with Workers AI (Llama 3.2 3B Instruct) when signed in. -- Sync the library through Convex after GitHub sign-in. Cloud records are - encrypted per user when the Cloudflare Worker secret `ENCRYPTION_MASTER_KEY` - is set; without it, records are synced in plaintext (`cryptoVersion 0`). +- Sync the library through Cloudflare D1 after GitHub sign-in. Cloud records + are encrypted per user using the existing `ENCRYPTION_MASTER_KEY` Worker + secret. Missing keys fail closed rather than uploading plaintext. A device downloads the whole library once, then only what changed since its - last sync (a server-clock cursor), which keeps Convex database I/O small. + last sync (a server-clock cursor), which keeps D1 database I/O small. - Use the app in English or Spanish, in light or dark theme, with keyboard shortcuts and a search palette. A service worker caches the app shell so the UI still loads offline. @@ -28,8 +28,9 @@ guest workspace that stays on the device you are using. ## Tech stack Next.js 16 (App Router, React 19) and Tailwind CSS 4, deployed as a Cloudflare -Worker through OpenNext. Convex provides the backend, the database, and GitHub -authentication; local snippet storage uses IndexedDB via Dexie. +Worker through OpenNext. D1 stores the workspace and Auth.js database sessions; +GitHub provides authentication. Local snippet storage uses IndexedDB via Dexie. +The private analytics console uses the same D1 database without a VPS. ## Requirements @@ -47,23 +48,25 @@ pnpm dev Open . This is enough to work on the guest workspace. -To develop cloud sync and authentication, run the Convex development backend in -a second terminal before starting the app: +To develop cloud sync, copy [.dev.vars.example](.dev.vars.example) to +`.dev.vars`, fill in a development OAuth app and encryption key, then run: ```bash pnpm dev:backend pnpm dev ``` -`pnpm dev:backend` writes the development deployment values to `.env.local`. -For encrypted cloud records, copy [.env.example](.env.example) to `.env` and set -`ENCRYPTION_MASTER_KEY`. GitHub sign-in also requires a GitHub OAuth app and -these Convex environment variables: +`pnpm dev:backend` applies D1 migrations locally. The GitHub callback is +`http://localhost:3000/api/auth/callback/github` for development and +`https://klipcode.com/api/auth/callback/github` for production. Use separate +OAuth applications for these environments. Missing OAuth credentials leave +the guest workspace available. -```bash -pnpm exec convex env set AUTH_GITHUB_ID -pnpm exec convex env set AUTH_GITHUB_SECRET -``` +Production uses `DB` as a D1 binding and Worker secrets `AUTH_SECRET`, +`AUTH_GITHUB_ID`, `AUTH_GITHUB_SECRET`, `AUTH_URL` and +`ENCRYPTION_MASTER_KEY`. Never replace the production master key: existing +wrapped keys and ciphertext depend on it. `pnpm deploy` applies versioned D1 +migrations before building and deploying the Worker. ## Checks and scripts @@ -82,7 +85,8 @@ CI runs all of the above on every push to `main` and on every pull request. ## Further reading - [Engineering audit](docs/audit/engineering-audit.md) -- [Convex backend](convex/) +- [D1 backend](src/server/) +- [Migration and rollback](docs/d1-migration.md) - [Environment example](.env.example) KlipCode is released under the [MIT License](LICENSE). diff --git a/convex.json b/convex.json new file mode 100644 index 0000000..a205825 --- /dev/null +++ b/convex.json @@ -0,0 +1,3 @@ +{ + "functions": "legacy/convex" +} diff --git a/convex/auth.ts b/convex/auth.ts deleted file mode 100644 index 0fc66c6..0000000 --- a/convex/auth.ts +++ /dev/null @@ -1,9 +0,0 @@ -import GitHub from "@auth/core/providers/github"; -import { convexAuth } from "@convex-dev/auth/server"; - -// GitHub is the only provider, matching the single "Sign in with GitHub" button -// in the aside. Credentials come from the Convex deployment env -// (AUTH_GITHUB_ID / AUTH_GITHUB_SECRET), never from the client bundle. -export const { auth, signIn, signOut, store, isAuthenticated } = convexAuth({ - providers: [GitHub], -}); diff --git a/docs/audit/engineering-audit.md b/docs/audit/engineering-audit.md index 7e1c319..218e5d1 100644 --- a/docs/audit/engineering-audit.md +++ b/docs/audit/engineering-audit.md @@ -1,5 +1,8 @@ # Engineering Audit +> Historical audit of the Convex architecture. The current D1 architecture and +> provider-separation change are documented in [D1 migration](../d1-migration.md). + **Audit date:** 2026-08-07 **Scope:** application code, Convex functions, local persistence, cloud sync, authentication, API routes, dependency graph, accessibility-sensitive UI paths, and a production-style browser smoke test. diff --git a/docs/d1-migration.md b/docs/d1-migration.md new file mode 100644 index 0000000..20dbff0 --- /dev/null +++ b/docs/d1-migration.md @@ -0,0 +1,110 @@ +# D1 migration + +The application and analytics console use Cloudflare Workers and the `klipcode` +D1 database. Production request paths have no Convex calls or VPS dependency. +GitHub accounts retain their existing user ids, so IndexedDB ownership and R2 +image paths remain valid. Ciphertext and wrapped DEKs are copied without +re-encryption. The master key stays the existing `klipcode` Worker secret. + +## Data and sessions + +- D1 workspace reads and writes include the authenticated owner in every query. +- Browser writes require the app's origin and expected account id; a cookie + change during sync cannot upload ciphertext into another account. +- Auth.js issues HTTP-only, Secure, SameSite cookies. Only hashes of opaque + session tokens are stored in D1. GitHub numeric identity ids resolve to the + imported users. Existing Convex sessions require GitHub sign-in again. +- Writes use indexed primary keys, last-write-wins timestamps and atomic D1 + batches. SQL triggers maintain deletion logs and analytics metadata. +- Pulls use a server-clock cursor, a one-minute overlap and a 30-day deletion + window. Old cursors and the pre-migration backend epoch force a full pull. +- Encryption failures stop sync and retain dirty local records for retry. +- D1 metadata and Worker secrets share a provider. Database exports still cannot + decrypt records alone, but this no longer provides separation of providers. + +## Repeatable copy + +Keep all exports and generated SQL outside Git, in a protected directory on +persistent storage or an external backup. Both contain private account data. +First test against local D1, then the remote database before routing app traffic. + +```bash +pnpm exec convex export --prod --path /protected/convex.zip +python3 scripts/migrate-convex-to-d1.py /protected/convex.zip \ + --analytics /protected/old-analytics-snapshot.json --sql /protected/import.sql +pnpm exec wrangler d1 migrations apply DB --remote +pnpm exec wrangler d1 execute DB --remote --file /protected/import.sql +python3 scripts/migrate-convex-to-d1.py /protected/convex.zip --verify +``` + +For the final frozen copy, use prepared REST batches rather than another bulk +SQL import. Cloudflare returned `D1_RESET_DO` on the large final SQL file during +the production migration. Prepared parameters avoid oversized SQL statements; +unchanged workspace records are skipped to preserve edit history and reduce +writes. The script refuses this path if D1 already contains login sessions. +Use a D1 API token from the environment or an existing protected Wrangler OAuth +configuration; never put a token in a command argument. +The copy script requires Python 3.11 or newer. Run its isolated SQLite/API tests +with `python3 scripts/test-migrate-convex-to-d1.py`. + +```bash +python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip \ + --apply-remote --source-frozen \ + --account-id YOUR_ACCOUNT_ID --database-id YOUR_DATABASE_ID \ + --oauth-config /protected/wrangler/config/default.toml +python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip --verify +``` + +The importer verifies every user id, provider identity, ciphertext field and +wrapped key. It is idempotent only **before cutover**: it also reconciles source +deletions. Never re-import over a live D1 workspace. + +Before the final copy, freeze the archived Convex workspace using its +`KLIPCODE_MIGRATED=true` environment guard. A stale browser then keeps its +unsynced changes locally until it reloads and signs in to the D1 version. +Disable the old VPS deploy timer and protect the production Worker from old +automatic deploy commands until the migration branch is published. Add the +new GitHub return URL before deploying; remove the old URL after validation. + +## Rollback + +Before D1 accepts user writes, restore the recorded pre-migration Worker +version, old GitHub callback and unset `KLIPCODE_MIGRATED` in Convex. Source data +is preserved. After D1 accepts writes, **do not switch back directly**: freeze D1 +writes, export it, reconcile new records/deletions into the source, and verify +before restoring traffic. Worker rollback alone cannot roll database state back. + +Convex code remains archived solely for the source freeze, verification and +rollback. It is not part of the deployed app backend. Remove the legacy project +and credentials only after the backup retention window and explicit approval. + +## Production cutover on 2026-09-30 + +The verified final export contains 14 users, 14 GitHub identities, 13 wrapped +keys, 43 folders and 302 snippets. App version +`6682cee2-f690-4748-a1d8-957e1fcbc996` and console version +`bf260137-d3d4-483c-8f59-934723ad4a04` completed the cutover. Production checks +confirmed GitHub sign-in resolves to an imported user, existing encrypted fields +decrypt, and encrypted create/edit/delete succeeds. The old GitHub callback was +removed and the Convex source remains frozen for recovery. + +The production Cloudflare build command currently uploads a version without +activating traffic. Once this migration is published to the production branch, +restore the deploy command to `pnpm deploy`, which applies D1 migrations before +building and deploying. Do not restore automatic activation while the branch +still contains the old Convex application. The VPS origin, tunnel and deploy +timer are disabled; infrastructure provisioning also keeps them disabled. + +The private console polls a single revision row every ten seconds while visible, +then loads cached metadata when the revision changes. It never reads ciphertext +for analytics and makes no Convex requests. + +## Legacy timestamp compatibility + +Imported records retain ISO timestamps with explicit offsets such as +`2026-07-16T11:04:39.716+00:00`, including variable fractional precision. The +backend accepts those alongside `Z` timestamps, preserves their original values, +and compares update timestamps by instant for last-write-wins. Version +`1f3b6e7d-69df-4ee1-b2a0-d08ba7bdabbf` fixes the initial validation regression +that rejected edits to these records. A real D1 regression test covers the +imported format and stale writes with a different UTC offset. diff --git a/e2e/mobile.spec.ts b/e2e/mobile.spec.ts index 698e2b7..4bec004 100644 --- a/e2e/mobile.spec.ts +++ b/e2e/mobile.spec.ts @@ -78,7 +78,9 @@ test("a swipe that falls short springs back", async ({ page }) => { test("a short flick still commits", async ({ page }) => { await gotoMobileApp(page); - await dragHorizontally(page, panel(page), { dx: -60 }); + // One native move keeps the flick fast even on a loaded CI/VPS host; eight + // separate protocol round trips can accidentally turn it into a slow drag. + await dragHorizontally(page, panel(page), { dx: -60, steps: 1 }); await expectSelected(page, "My Space"); }); diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a7b118..47b9c6a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -21,7 +21,7 @@ const eslintConfig = defineConfig([ // Isolated remote dev output (generated by KLIPCODE_DIST_DIR). ".next-remote/**", // Auto-generated by `npx convex dev`; not meant to be linted. - "convex/_generated/**", + "legacy/convex/_generated/**", ]), ]); diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..b096b4f --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,10 @@ +# Legacy source backend + +This Convex project is retained only for migration verification, the source +write freeze and rollback. No deployed application route imports it. Production +is `src/server` and `migrations`, using Workers bindings and D1. + +`KLIPCODE_MIGRATED=true` blocks legacy workspace reads and writes. Unset it only +when rolling back before D1 accepts new user writes, or after reconciling D1. +The CLI functions directory is configured in the root `convex.json` solely to +keep recovery commands repeatable. Convex packages are development dependencies. diff --git a/convex/_generated/api.d.ts b/legacy/convex/_generated/api.d.ts similarity index 100% rename from convex/_generated/api.d.ts rename to legacy/convex/_generated/api.d.ts diff --git a/convex/_generated/api.js b/legacy/convex/_generated/api.js similarity index 100% rename from convex/_generated/api.js rename to legacy/convex/_generated/api.js diff --git a/convex/_generated/dataModel.d.ts b/legacy/convex/_generated/dataModel.d.ts similarity index 100% rename from convex/_generated/dataModel.d.ts rename to legacy/convex/_generated/dataModel.d.ts diff --git a/convex/_generated/server.d.ts b/legacy/convex/_generated/server.d.ts similarity index 100% rename from convex/_generated/server.d.ts rename to legacy/convex/_generated/server.d.ts diff --git a/convex/_generated/server.js b/legacy/convex/_generated/server.js similarity index 100% rename from convex/_generated/server.js rename to legacy/convex/_generated/server.js diff --git a/convex/analytics.test.ts b/legacy/convex/analytics.test.ts similarity index 100% rename from convex/analytics.test.ts rename to legacy/convex/analytics.test.ts diff --git a/convex/analytics.ts b/legacy/convex/analytics.ts similarity index 100% rename from convex/analytics.ts rename to legacy/convex/analytics.ts diff --git a/convex/auth.config.ts b/legacy/convex/auth.config.ts similarity index 100% rename from convex/auth.config.ts rename to legacy/convex/auth.config.ts diff --git a/legacy/convex/auth.ts b/legacy/convex/auth.ts new file mode 100644 index 0000000..fae8f3a --- /dev/null +++ b/legacy/convex/auth.ts @@ -0,0 +1,37 @@ +import GitHub from "@auth/core/providers/github"; +import type { ActionCtx } from "./_generated/server"; +import { action } from "./_generated/server"; +import { v } from "convex/values"; +import { convexAuth } from "@convex-dev/auth/server"; + +// GitHub is the only provider, matching the single "Sign in with GitHub" button +// in the aside. Credentials come from the Convex deployment env +// (AUTH_GITHUB_ID / AUTH_GITHUB_SECRET), never from the client bundle. +const legacy = convexAuth({ + providers: [GitHub], +}); + +export const { auth, signOut, store, isAuthenticated } = legacy; +// Stop stale tabs from refreshing legacy tokens after the final copy. +export const signIn = action({ + args: { + provider: v.optional(v.string()), + params: v.optional(v.any()), + verifier: v.optional(v.string()), + refreshToken: v.optional(v.string()), + calledBy: v.optional(v.string()), + }, + handler: async (ctx, args) => { + if (process.env.KLIPCODE_MIGRATED === "true") + throw new Error( + "KlipCode has moved. Reload and sign in at klipcode.com.", + ); + const source = legacy.signIn as unknown as { + _handler: ( + ctx: ActionCtx, + args: Record, + ) => Promise; + }; + return source._handler(ctx, args); + }, +}); diff --git a/convex/http.ts b/legacy/convex/http.ts similarity index 74% rename from convex/http.ts rename to legacy/convex/http.ts index dfb3683..342953d 100644 --- a/convex/http.ts +++ b/legacy/convex/http.ts @@ -5,6 +5,6 @@ import { auth } from "./auth"; const http = httpRouter(); // Mounts the OAuth callback routes (/api/auth/*) on the deployment's .site domain. -auth.addHttpRoutes(http); +if (process.env.KLIPCODE_MIGRATED !== "true") auth.addHttpRoutes(http); export default http; diff --git a/convex/lib/auth.ts b/legacy/convex/lib/auth.ts similarity index 83% rename from convex/lib/auth.ts rename to legacy/convex/lib/auth.ts index 6b3e400..ed6b13c 100644 --- a/convex/lib/auth.ts +++ b/legacy/convex/lib/auth.ts @@ -13,6 +13,9 @@ type AuthCtx = { auth: { getUserIdentity: () => Promise<{ subject: string } | nu * only the first segment identifies the account. */ export async function requireUserId(ctx: AuthCtx): Promise> { + if (process.env.KLIPCODE_MIGRATED === "true") { + throw new Error("KlipCode has moved. Reload the app and sign in again; unsynced changes remain on this device."); + } const identity = await ctx.auth.getUserIdentity(); if (!identity) { diff --git a/convex/lib/hierarchy.ts b/legacy/convex/lib/hierarchy.ts similarity index 100% rename from convex/lib/hierarchy.ts rename to legacy/convex/lib/hierarchy.ts diff --git a/convex/lib/sync.ts b/legacy/convex/lib/sync.ts similarity index 100% rename from convex/lib/sync.ts rename to legacy/convex/lib/sync.ts diff --git a/convex/lib/syncCursor.ts b/legacy/convex/lib/syncCursor.ts similarity index 100% rename from convex/lib/syncCursor.ts rename to legacy/convex/lib/syncCursor.ts diff --git a/convex/lib/workspaceWrites.ts b/legacy/convex/lib/workspaceWrites.ts similarity index 100% rename from convex/lib/workspaceWrites.ts rename to legacy/convex/lib/workspaceWrites.ts diff --git a/convex/migrations.ts b/legacy/convex/migrations.ts similarity index 100% rename from convex/migrations.ts rename to legacy/convex/migrations.ts diff --git a/convex/schema.ts b/legacy/convex/schema.ts similarity index 100% rename from convex/schema.ts rename to legacy/convex/schema.ts diff --git a/convex/tsconfig.json b/legacy/convex/tsconfig.json similarity index 100% rename from convex/tsconfig.json rename to legacy/convex/tsconfig.json diff --git a/convex/userKeys.ts b/legacy/convex/userKeys.ts similarity index 100% rename from convex/userKeys.ts rename to legacy/convex/userKeys.ts diff --git a/convex/users.ts b/legacy/convex/users.ts similarity index 100% rename from convex/users.ts rename to legacy/convex/users.ts diff --git a/convex/workspace.test.ts b/legacy/convex/workspace.test.ts similarity index 100% rename from convex/workspace.test.ts rename to legacy/convex/workspace.test.ts diff --git a/convex/workspace.ts b/legacy/convex/workspace.ts similarity index 100% rename from convex/workspace.ts rename to legacy/convex/workspace.ts diff --git a/migrations/0001_d1_backend.sql b/migrations/0001_d1_backend.sql new file mode 100644 index 0000000..e8f2be0 --- /dev/null +++ b/migrations/0001_d1_backend.sql @@ -0,0 +1,168 @@ +-- Auth.js tables and KlipCode workspace. Applied explicitly with Wrangler. +CREATE TABLE IF NOT EXISTS "accounts" ( + "id" text NOT NULL, + "userId" text NOT NULL DEFAULT NULL, + "type" text NOT NULL DEFAULT NULL, + "provider" text NOT NULL DEFAULT NULL, + "providerAccountId" text NOT NULL DEFAULT NULL, + "refresh_token" text DEFAULT NULL, + "access_token" text DEFAULT NULL, + "expires_at" number DEFAULT NULL, + "token_type" text DEFAULT NULL, + "scope" text DEFAULT NULL, + "id_token" text DEFAULT NULL, + "session_state" text DEFAULT NULL, + "oauth_token_secret" text DEFAULT NULL, + "oauth_token" text DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS "sessions" ( + "id" text NOT NULL, + "sessionToken" text NOT NULL, + "userId" text NOT NULL DEFAULT NULL, + "expires" datetime NOT NULL DEFAULT NULL, + PRIMARY KEY (sessionToken) +); + +CREATE TABLE IF NOT EXISTS "users" ( + "id" text NOT NULL DEFAULT '', + "name" text DEFAULT NULL, + "email" text DEFAULT NULL, + "emailVerified" datetime DEFAULT NULL, + "image" text DEFAULT NULL, + PRIMARY KEY (id) +); + +CREATE TABLE IF NOT EXISTS "verification_tokens" ( + "identifier" text NOT NULL, + "token" text NOT NULL DEFAULT NULL, + "expires" datetime NOT NULL DEFAULT NULL, + PRIMARY KEY (token) +); +ALTER TABLE users ADD COLUMN created_at INTEGER NOT NULL DEFAULT 0; +ALTER TABLE users ADD COLUMN country TEXT; +CREATE UNIQUE INDEX accounts_provider_identity ON accounts(provider, providerAccountId); +CREATE INDEX accounts_user ON accounts(userId); +CREATE INDEX users_email ON users(email); +CREATE INDEX sessions_user ON sessions(userId); +CREATE TABLE user_keys (user_id TEXT PRIMARY KEY REFERENCES users(id), wrapped_dek TEXT NOT NULL); +CREATE TABLE analytics_state (id INTEGER PRIMARY KEY CHECK(id = 1), revision INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL DEFAULT 0); +INSERT INTO analytics_state(id) VALUES (1); +CREATE TABLE analytics_signins (id TEXT PRIMARY KEY, account TEXT NOT NULL, at INTEGER NOT NULL); +CREATE INDEX analytics_signins_account ON analytics_signins(account, at); +CREATE TABLE analytics_edits (snippet TEXT NOT NULL, account TEXT NOT NULL, language TEXT NOT NULL, at INTEGER NOT NULL, PRIMARY KEY(snippet, at)); +CREATE INDEX analytics_edits_account ON analytics_edits(account, at); +CREATE TABLE analytics_snippets (id TEXT PRIMARY KEY, account TEXT NOT NULL, language TEXT NOT NULL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, deleted_at TEXT, pinned INTEGER NOT NULL, in_folder INTEGER NOT NULL); +CREATE INDEX analytics_snippets_account ON analytics_snippets(account); +CREATE TABLE analytics_folders (id TEXT PRIMARY KEY, account TEXT NOT NULL, created_at TEXT NOT NULL); +CREATE INDEX analytics_folders_account ON analytics_folders(account); +CREATE TABLE deletions (owner_id TEXT NOT NULL, kind TEXT NOT NULL CHECK(kind IN ('folder','snippet')), client_id TEXT NOT NULL, deleted_at INTEGER NOT NULL, PRIMARY KEY(owner_id, kind, client_id)); +CREATE INDEX deletions_owner_time ON deletions(owner_id, deleted_at); + +CREATE TABLE folders ( + owner_id TEXT NOT NULL REFERENCES users(id), client_id TEXT NOT NULL, + analytics_id TEXT NOT NULL UNIQUE, data TEXT NOT NULL CHECK(json_valid(data)), + updated_at TEXT NOT NULL, server_updated_at INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)), + PRIMARY KEY(owner_id, client_id), CHECK(length(CAST(data AS BLOB)) <= 1900000) +); +CREATE INDEX folders_owner_time ON folders(owner_id, server_updated_at); + +CREATE TABLE snippets ( + owner_id TEXT NOT NULL REFERENCES users(id), client_id TEXT NOT NULL, + analytics_id TEXT NOT NULL UNIQUE, data TEXT NOT NULL CHECK(json_valid(data)), + updated_at TEXT NOT NULL, server_updated_at INTEGER NOT NULL DEFAULT (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)), + PRIMARY KEY(owner_id, client_id), CHECK(length(CAST(data AS BLOB)) <= 1900000) +); +CREATE INDEX snippets_owner_time ON snippets(owner_id, server_updated_at); +CREATE INDEX folders_parent ON folders(owner_id, json_extract(data, '$.parentId')); +CREATE INDEX snippets_folder ON snippets(owner_id, json_extract(data, '$.folderId')); + +CREATE TRIGGER folders_cycle_insert AFTER INSERT ON folders BEGIN + SELECT RAISE(ABORT, 'Folder hierarchy cannot contain cycles') WHERE EXISTS ( + WITH RECURSIVE ancestors(client_id) AS ( + SELECT json_extract(NEW.data, '$.parentId') + UNION + SELECT json_extract(f.data, '$.parentId') FROM folders f JOIN ancestors a ON f.client_id = a.client_id WHERE f.owner_id = NEW.owner_id + ) SELECT 1 FROM ancestors WHERE client_id = NEW.client_id + ); +END; + +CREATE TRIGGER folders_cycle_update AFTER UPDATE ON folders BEGIN + SELECT RAISE(ABORT, 'Folder hierarchy cannot contain cycles') WHERE EXISTS ( + WITH RECURSIVE ancestors(client_id) AS ( + SELECT json_extract(NEW.data, '$.parentId') + UNION + SELECT json_extract(f.data, '$.parentId') FROM folders f JOIN ancestors a ON f.client_id = a.client_id WHERE f.owner_id = NEW.owner_id + ) SELECT 1 FROM ancestors WHERE client_id = NEW.client_id + ); +END; + +CREATE TRIGGER folders_analytics_insert AFTER INSERT ON folders BEGIN + INSERT INTO analytics_folders(id, account, created_at) VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data, '$.createdAt')) ON CONFLICT(id) DO UPDATE SET created_at=excluded.created_at; + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='folder' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_analytics_update AFTER UPDATE ON folders BEGIN + INSERT INTO analytics_folders(id, account, created_at) VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data, '$.createdAt')) ON CONFLICT(id) DO UPDATE SET created_at=excluded.created_at; + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='folder' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_delete AFTER DELETE ON folders BEGIN + INSERT INTO deletions(owner_id, kind, client_id, deleted_at) VALUES (OLD.owner_id,'folder',OLD.client_id,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ON CONFLICT(owner_id,kind,client_id) DO UPDATE SET deleted_at=excluded.deleted_at; + DELETE FROM analytics_folders WHERE id=OLD.analytics_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_analytics_insert AFTER INSERT ON snippets BEGIN + INSERT INTO analytics_snippets(id, account, language, created_at, updated_at, deleted_at, pinned, in_folder) +VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), json_extract(NEW.data,'$.createdAt'), json_extract(NEW.data,'$.updatedAt'), json_extract(NEW.data,'$.deletedAt'), json_extract(NEW.data,'$.isPinnedAside') OR json_extract(NEW.data,'$.isPinnedHome'), json_extract(NEW.data,'$.folderId') IS NOT NULL) +ON CONFLICT(id) DO UPDATE SET language=excluded.language, created_at=excluded.created_at, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, pinned=excluded.pinned, in_folder=excluded.in_folder; +INSERT OR IGNORE INTO analytics_edits(snippet, account, language, at) +SELECT NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), CAST(round((julianday(json_extract(NEW.data,'$.updatedAt'))-2440587.5)*86400000) AS INTEGER) +WHERE (julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.createdAt')))*86400000 > 60000 +AND (json_extract(NEW.data,'$.deletedAt') IS NULL OR abs((julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.deletedAt')))*86400000) >= 5000); + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='snippet' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_analytics_update AFTER UPDATE ON snippets BEGIN + INSERT INTO analytics_snippets(id, account, language, created_at, updated_at, deleted_at, pinned, in_folder) +VALUES (NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), json_extract(NEW.data,'$.createdAt'), json_extract(NEW.data,'$.updatedAt'), json_extract(NEW.data,'$.deletedAt'), json_extract(NEW.data,'$.isPinnedAside') OR json_extract(NEW.data,'$.isPinnedHome'), json_extract(NEW.data,'$.folderId') IS NOT NULL) +ON CONFLICT(id) DO UPDATE SET language=excluded.language, created_at=excluded.created_at, updated_at=excluded.updated_at, deleted_at=excluded.deleted_at, pinned=excluded.pinned, in_folder=excluded.in_folder; +INSERT OR IGNORE INTO analytics_edits(snippet, account, language, at) +SELECT NEW.analytics_id, NEW.owner_id, json_extract(NEW.data,'$.language'), CAST(round((julianday(json_extract(NEW.data,'$.updatedAt'))-2440587.5)*86400000) AS INTEGER) +WHERE (julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.createdAt')))*86400000 > 60000 +AND (json_extract(NEW.data,'$.deletedAt') IS NULL OR abs((julianday(json_extract(NEW.data,'$.updatedAt'))-julianday(json_extract(NEW.data,'$.deletedAt')))*86400000) >= 5000); + DELETE FROM deletions WHERE owner_id=NEW.owner_id AND kind='snippet' AND client_id=NEW.client_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER snippets_delete AFTER DELETE ON snippets BEGIN + INSERT INTO deletions(owner_id, kind, client_id, deleted_at) VALUES (OLD.owner_id,'snippet',OLD.client_id,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) + ON CONFLICT(owner_id,kind,client_id) DO UPDATE SET deleted_at=excluded.deleted_at; + DELETE FROM analytics_snippets WHERE id=OLD.analytics_id; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TRIGGER folders_detach BEFORE DELETE ON folders BEGIN + UPDATE snippets SET data=json_set(data,'$.folderId',NULL), server_updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) + WHERE owner_id=OLD.owner_id AND json_extract(data,'$.folderId')=OLD.client_id; +END; +CREATE TRIGGER users_created AFTER INSERT ON users BEGIN + UPDATE users SET created_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=NEW.id AND NEW.created_at=0; + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; +CREATE TRIGGER sessions_created AFTER INSERT ON sessions BEGIN + INSERT OR IGNORE INTO analytics_signins(id,account,at) VALUES(NEW.id,NEW.userId,CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)); + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; +CREATE TRIGGER accounts_created AFTER INSERT ON accounts BEGIN + UPDATE analytics_state SET revision=revision+1, updated_at=CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) WHERE id=1; +END; + +CREATE TABLE app_meta(key TEXT PRIMARY KEY, value INTEGER NOT NULL); +INSERT INTO app_meta VALUES('backend_epoch',CAST((julianday('now')-2440587.5)*86400000 AS INTEGER)); diff --git a/migrations/0002_signin_deduplication.sql b/migrations/0002_signin_deduplication.sql new file mode 100644 index 0000000..e06de1f --- /dev/null +++ b/migrations/0002_signin_deduplication.sql @@ -0,0 +1,3 @@ +-- Legacy snapshots and the final export can describe the same sign-in. +DELETE FROM analytics_signins WHERE rowid NOT IN (SELECT MIN(rowid) FROM analytics_signins GROUP BY account,at); +CREATE UNIQUE INDEX analytics_signins_unique_event ON analytics_signins(account,at); diff --git a/package.json b/package.json index e6a99c5..77c92a5 100644 --- a/package.json +++ b/package.json @@ -7,21 +7,23 @@ "scripts": { "build": "next build", "dev": "next dev", - "dev:backend": "convex dev", + "dev:backend": "wrangler d1 migrations apply DB --local", "start": "next start", "lint": "eslint", "test": "vitest run", "test:watch": "vitest", "test:e2e": "playwright test", "prepare": "husky", - "postinstall": "wrangler types --env-interface CloudflareEnv cloudflare-env.d.ts", + "postinstall": "pnpm cf-typegen", "preview": "opennextjs-cloudflare build && opennextjs-cloudflare preview", - "deploy": "convex deploy --cmd 'opennextjs-cloudflare build' && opennextjs-cloudflare deploy", - "upload": "convex deploy --cmd 'opennextjs-cloudflare build' && opennextjs-cloudflare upload", - "cf-typegen": "wrangler types --env-interface CloudflareEnv cloudflare-env.d.ts" + "deploy": "pnpm db:migrate && opennextjs-cloudflare build && opennextjs-cloudflare deploy", + "upload": "opennextjs-cloudflare build && opennextjs-cloudflare upload", + "cf-typegen": "wrangler types --env-file .dev.vars.example --env-interface CloudflareEnv cloudflare-env.d.ts", + "db:migrate": "wrangler d1 migrations apply DB --remote" }, "dependencies": { "@auth/core": "0.41.3", + "@auth/d1-adapter": "^1.11.3", "@codemirror/autocomplete": "^6.20.3", "@codemirror/lang-cpp": "^6.0.3", "@codemirror/lang-css": "^6.3.1", @@ -38,7 +40,6 @@ "@codemirror/language": "^6.12.3", "@codemirror/legacy-modes": "^6.5.2", "@codemirror/state": "^6.6.0", - "@convex-dev/auth": "^0.0.94", "@lezer/highlight": "^1.2.3", "@opennextjs/cloudflare": "^1.20.2", "@replit/codemirror-lang-svelte": "^6.0.0", @@ -60,7 +61,6 @@ "@uiw/codemirror-theme-vscode": "^4.25.9", "@uiw/react-codemirror": "^4.25.8", "clsx": "^2.1.1", - "convex": "^1.43.0", "dexie": "^4.3.0", "lowlight": "^3.3.0", "lucide-react": "^1.6.0", @@ -73,6 +73,7 @@ "tiptap-markdown": "^0.9.0" }, "devDependencies": { + "@convex-dev/auth": "^0.0.94", "@edge-runtime/vm": "^5.0.0", "@playwright/test": "^1.61.1", "@tailwindcss/postcss": "^4", @@ -81,12 +82,14 @@ "@types/react": "^19", "@types/react-dom": "^19", "babel-plugin-react-compiler": "1.0.0", + "convex": "^1.43.0", "convex-test": "^0.0.54", "eslint": "^9", "eslint-config-next": "16.2.11", "fake-indexeddb": "^6.2.5", "husky": "^9.1.7", "jsdom": "^30.0.1", + "miniflare": "4.20260708.1", "tailwindcss": "^4", "typescript": "^5", "vite": "^8.2.0", diff --git a/playwright.config.ts b/playwright.config.ts index a68d1c8..451a762 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -5,11 +5,13 @@ import { defineConfig, devices } from "@playwright/test"; * * Cloud-sync behaviour is covered where it actually lives rather than through * the browser: `src/__tests__/sync.test.ts` drives the client sync engine - * against an in-memory deployment, and `convex/workspace.test.ts` runs the real + * against an in-memory deployment, and `src/server/workspace.test.ts` runs the real * backend functions (ownership, last-write-wins, cycle rejection, delete - * cascade) on Convex's test harness. Both run in CI with no Docker and no live + * cascade) on local D1. Both run in CI with no Docker and no live * deployment — which the previous Supabase-backed sync suite could not do. */ +const testPort = Number(process.env.KLIPCODE_TEST_PORT ?? 3000); + export default defineConfig({ testDir: "e2e", fullyParallel: true, @@ -17,7 +19,7 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, reporter: process.env.CI ? "github" : "list", use: { - baseURL: "http://localhost:3000", + baseURL: `http://127.0.0.1:${testPort}`, trace: "on-first-retry", }, projects: [ @@ -34,8 +36,8 @@ export default defineConfig({ ], webServer: [ { - command: "pnpm dev", - url: "http://localhost:3000", + command: `pnpm exec next dev --hostname 127.0.0.1 --port ${testPort}`, + url: `http://127.0.0.1:${testPort}`, reuseExistingServer: !process.env.CI, timeout: 120_000, }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2f4d80..791713c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: '@auth/core': specifier: 0.41.3 version: 0.41.3 + '@auth/d1-adapter': + specifier: ^1.11.3 + version: 1.11.3 '@codemirror/autocomplete': specifier: ^6.20.3 version: 6.20.3 @@ -59,9 +62,6 @@ importers: '@codemirror/state': specifier: ^6.6.0 version: 6.6.0 - '@convex-dev/auth': - specifier: ^0.0.94 - version: 0.0.94(@auth/core@0.41.3)(convex@1.43.0(react@19.2.4))(react@19.2.4) '@lezer/highlight': specifier: ^1.2.3 version: 1.2.3 @@ -125,9 +125,6 @@ importers: clsx: specifier: ^2.1.1 version: 2.1.1 - convex: - specifier: ^1.43.0 - version: 1.43.0(react@19.2.4) dexie: specifier: ^4.3.0 version: 4.3.0 @@ -159,6 +156,9 @@ importers: specifier: ^0.9.0 version: 0.9.0(@tiptap/core@2.27.2(@tiptap/pm@2.27.2)) devDependencies: + '@convex-dev/auth': + specifier: ^0.0.94 + version: 0.0.94(@auth/core@0.41.3)(convex@1.43.0(react@19.2.4))(react@19.2.4) '@edge-runtime/vm': specifier: ^5.0.0 version: 5.0.0 @@ -183,6 +183,9 @@ importers: babel-plugin-react-compiler: specifier: 1.0.0 version: 1.0.0 + convex: + specifier: ^1.43.0 + version: 1.43.0(react@19.2.4) convex-test: specifier: ^0.0.54 version: 0.0.54(convex@1.43.0(react@19.2.4)) @@ -201,6 +204,9 @@ importers: jsdom: specifier: ^30.0.1 version: 30.0.1(@noble/hashes@1.8.0) + miniflare: + specifier: 4.20260708.1 + version: 4.20260708.1 tailwindcss: specifier: ^4 version: 4.2.2 @@ -303,6 +309,9 @@ packages: nodemailer: optional: true + '@auth/d1-adapter@1.11.3': + resolution: {integrity: sha512-F71eQVTJzP0igCaPMFqE7EUp62sea6COGQs+6rF7kN1998KBRKy+I0xennOz00gnL/dMOM82PP0GvE5Vtpjm7g==} + '@aws-crypto/sha1-browser@5.2.0': resolution: {integrity: sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==} @@ -5088,6 +5097,14 @@ snapshots: preact: 10.24.3 preact-render-to-string: 6.5.11(preact@10.24.3) + '@auth/d1-adapter@1.11.3': + dependencies: + '@auth/core': 0.41.3 + transitivePeerDependencies: + - '@simplewebauthn/browser' + - '@simplewebauthn/server' + - nodemailer + '@aws-crypto/sha1-browser@5.2.0': dependencies: '@aws-crypto/supports-web-crypto': 5.2.0 diff --git a/public/sw.js b/public/sw.js index 744eec4..2c3007e 100644 --- a/public/sw.js +++ b/public/sw.js @@ -13,7 +13,7 @@ * * Bump CACHE_VERSION to invalidate previously cached responses. */ -const CACHE_VERSION = "v1"; +const CACHE_VERSION = "v2"; const CACHE_NAME = `klipcode-${CACHE_VERSION}`; const APP_SHELL = "/app"; diff --git a/scripts/migrate-convex-to-d1.py b/scripts/migrate-convex-to-d1.py new file mode 100644 index 0000000..0ed0e73 --- /dev/null +++ b/scripts/migrate-convex-to-d1.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Generate an idempotent D1 import from a protected Convex export. Never decrypts. + +Run only before the D1 cutover, with Convex workspace writes frozen. SQL files +contain private data: keep them outside the repository, with mode 0600. +""" +import argparse +import hashlib +import json +import os +import subprocess +import urllib.request +import urllib.error +import tomllib +import zipfile +from pathlib import Path +from datetime import datetime, timezone + +parser = argparse.ArgumentParser() +parser.add_argument('export', type=Path) +parser.add_argument('--sql', type=Path) +parser.add_argument('--analytics', type=Path) +parser.add_argument('--verify', action='store_true') +parser.add_argument('--local', action='store_true') +parser.add_argument('--apply-remote', action='store_true', help='Prepared REST batches; only before cutover') +parser.add_argument('--source-frozen', action='store_true', help='Confirm the source write guard is enabled') +parser.add_argument('--account-id') +parser.add_argument('--database-id') +parser.add_argument('--oauth-config', type=Path, help='Wrangler OAuth config; otherwise use CLOUDFLARE_API_TOKEN') +args = parser.parse_args() +archive = zipfile.ZipFile(args.export) +def table(name): + try: return [json.loads(line) for line in archive.read(name+'/documents.jsonl').splitlines() if line] + except KeyError: return [] +def quote(value): + if value is None: return 'NULL' + if isinstance(value, (int,float)): return str(round(value)) + return "'"+str(value).replace("'","''")+"'" +def iso(ms): return datetime.fromtimestamp(ms/1000, timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z') +def hashid(s): return hashlib.sha256(s.encode()).hexdigest()[:8] +users=table('users'); keys=table('userKeys'); folders=table('folders'); snippets=table('snippets') +clock=round(datetime.now(timezone.utc).timestamp()*1000) +statements=[] +prepared=[] +def insert(name, fields, values, suffix=''): + statements.append(f"INSERT INTO {name}({','.join(fields)}) VALUES({','.join(quote(x) for x in values)}) {suffix};") + prepared.append((name, {'sql':f"INSERT INTO {name}({','.join(fields)}) VALUES({','.join('?' for _ in values)}) {suffix}", 'params':values})) +for u in users: + insert('users',['id','name','email','image','emailVerified','created_at','country'],[u['_id'],u.get('name'),u.get('email'),u.get('image'),iso(u['emailVerificationTime']) if u.get('emailVerificationTime') else None,round(u['_creationTime']),u.get('country')], 'ON CONFLICT(id) DO UPDATE SET name=excluded.name,email=excluded.email,image=excluded.image,emailVerified=excluded.emailVerified') +for a in table('authAccounts'): + if a['provider']!='github': raise ValueError('Unsupported legacy provider') + insert('accounts',['id','userId','type','provider','providerAccountId'],[a['_id'],a['userId'],'oauth',a['provider'],a['providerAccountId']], 'ON CONFLICT(provider,providerAccountId) DO NOTHING') +for k in keys: + insert('user_keys',['user_id','wrapped_dek'],[k['userId'],k['wrappedDek']], 'ON CONFLICT(user_id) DO NOTHING') +common=['clientId','createdAt','updatedAt','deletedAt','cryptoVersion','isPinnedAside','isPinnedHome'] +def wire(row,kind): + return {k:row[k] for k in common+(['name','parentId'] if kind=='folders' else ['title','code','language','folderId'])} +for name,records in [('folders',folders),('snippets',snippets)]: + for r in records: + insert(name,['owner_id','client_id','analytics_id','data','updated_at','server_updated_at'],[r['ownerId'],r['clientId'],r['_id'],json.dumps(wire(r,name),separators=(',',':'),ensure_ascii=True),r['updatedAt'],clock], 'ON CONFLICT(owner_id,client_id) DO UPDATE SET data=excluded.data,updated_at=excluded.updated_at,server_updated_at=excluded.server_updated_at WHERE excluded.updated_at>='+name+'.updated_at AND excluded.data<>'+name+'.data') + # A second pre-cutover import reconciles deletions made since the first copy. + present=json.dumps([r['_id'] for r in records]) + statements.append(f'DELETE FROM {name} WHERE analytics_id NOT IN (SELECT value FROM json_each({quote(present)}));') + prepared.append((name, {'sql':f'DELETE FROM {name} WHERE analytics_id NOT IN (SELECT value FROM json_each(?))', 'params':[present]})) +for s in table('authSessions'): + insert('analytics_signins',['id','account','at'],[s['_id'],s['userId'],round(s['_creationTime'])], 'ON CONFLICT DO NOTHING') +if args.analytics: + previous=json.loads(args.analytics.read_text()) + owners={hashid(u['_id']):u['_id'] for u in users} + snippet_ids={hashid(s['_id']):s['_id'] for s in snippets} + for e in previous.get('edits',[]): + if e['account'] in owners: + insert('analytics_edits',['snippet','account','language','at'],[snippet_ids.get(e['snippet'],'anon:'+e['snippet']),owners[e['account']],e['language'],e['at']], 'ON CONFLICT(snippet,at) DO NOTHING') + for s in previous.get('sessions',[]): + if s['account'] in owners: + insert('analytics_signins',['id','account','at'],['legacy:'+s['account']+':'+str(s['at']),owners[s['account']],s['at']], 'ON CONFLICT DO NOTHING') +# Only the final copy's epoch is trusted; old device cursors force a full pull. +statements.append(f"UPDATE app_meta SET value={clock} WHERE key='backend_epoch';") +prepared.append(('app_meta', {'sql':"UPDATE app_meta SET value=? WHERE key='backend_epoch'", 'params':[clock]})) +if args.sql: + fd=os.open(args.sql,os.O_WRONLY|os.O_CREAT|os.O_TRUNC,0o600) + with os.fdopen(fd,'w') as f: f.write('\n'.join(statements)+'\n') + print(json.dumps({'users':len(users),'keys':len(keys),'folders':len(folders),'snippets':len(snippets),'statements':len(statements),'sql_bytes':args.sql.stat().st_size})) +if args.apply_remote: + if args.local or not args.source_frozen or not args.account_id or not args.database_id: + parser.error('--apply-remote requires --source-frozen, --account-id and --database-id, without --local') + token=os.environ.get('CLOUDFLARE_API_TOKEN') + if not token and args.oauth_config: + token=tomllib.loads(args.oauth_config.read_text()).get('oauth_token') + if not token: + parser.error('Provide CLOUDFLARE_API_TOKEN or --oauth-config; never pass tokens on the command line') + endpoint=f'https://api.cloudflare.com/client/v4/accounts/{args.account_id}/d1/database/{args.database_id}/query' + def remote(body): + request=urllib.request.Request(endpoint, data=json.dumps(body).encode(), headers={'Authorization':'Bearer '+token,'Content-Type':'application/json'}) + try: + with urllib.request.urlopen(request,timeout=60) as response: + result=json.load(response) + except urllib.error.HTTPError as error: + # API errors may echo private SQL parameters; report only the code. + raise RuntimeError(f'Cloudflare HTTP {error.code}; import stopped, do not activate traffic') from None + if not result.get('success') or any(not r.get('success') for r in result.get('result',[])): + raise RuntimeError('D1 import failed; do not activate traffic') + return result['result'] + if remote({'sql':'SELECT count(*) AS count FROM sessions'})[0]['results'][0]['count']: + raise RuntimeError('D1 already has login sessions. Refusing to import over a live workspace.') + stored={} + for kind in ['folders','snippets']: + stored[kind]={(r['owner_id'],r['client_id']):r for r in remote({'sql':f'SELECT owner_id,client_id,analytics_id,data FROM {kind}'})[0]['results']} + pending=[] + skipped=0 + for kind, statement in prepared: + if kind in stored and statement['sql'].startswith('INSERT'): + owner,client,analytics_id,data,*_=statement['params'] + old=stored[kind].get((owner,client)) + if old and old['analytics_id']==analytics_id and json.loads(old['data'])==json.loads(data): + skipped+=1 + continue + pending.append(statement) + # Large SQL imports can fail with D1_RESET_DO. Bind values instead, skipping + # unchanged ciphertext and bounding each request. The source stays frozen; + # verification must pass before any version accepts D1 traffic. + batch=[] + total=0 + size=0 + for statement in pending: + length=len(json.dumps(statement).encode()) + if batch and (len(batch)>=100 or size+length>2000000): + remote({'batch':batch}) + total+=len(batch) + batch=[] + size=0 + batch.append(statement) + size+=length + if batch: + remote({'batch':batch}) + total+=len(batch) + print(json.dumps({'applied_statements':total,'unchanged_workspace_records_skipped':skipped})) +if args.verify: + def query(sql): + proc=subprocess.run(['pnpm','exec','wrangler','d1','execute','DB','--local' if args.local else '--remote','--json','--command',sql],capture_output=True,text=True,check=True) + return json.loads(proc.stdout)[0]['results'] + stored_users=query('SELECT id FROM users'); assert {u['id'] for u in stored_users}=={u['_id'] for u in users},'User mismatch' + stored_keys=query('SELECT user_id,wrapped_dek FROM user_keys');assert {k['user_id']:k['wrapped_dek'] for k in stored_keys}=={k['userId']:k['wrappedDek'] for k in keys},'Key mismatch' + for kind, records in [('folders',folders),('snippets',snippets)]: + stored=query(f'SELECT owner_id,client_id,analytics_id,data FROM {kind}') + expected={(r['ownerId'],r['clientId']):(r['_id'],wire(r,kind)) for r in records} + actual={(r['owner_id'],r['client_id']):(r['analytics_id'],json.loads(r['data'])) for r in stored} + assert actual==expected,kind+' mismatch' + accounts=query('SELECT userId,provider,providerAccountId FROM accounts') + assert {(a['userId'],a['provider'],a['providerAccountId']) for a in accounts}=={(a['userId'],a['provider'],a['providerAccountId']) for a in table('authAccounts')},'GitHub account mismatch' + print('Verified all account ids, GitHub identities, ciphertext fields and wrapped keys byte for byte.') diff --git a/scripts/test-migrate-convex-to-d1.py b/scripts/test-migrate-convex-to-d1.py new file mode 100644 index 0000000..e59e802 --- /dev/null +++ b/scripts/test-migrate-convex-to-d1.py @@ -0,0 +1,49 @@ +import io, json, runpy, sqlite3, tempfile, unittest, zipfile +from pathlib import Path +from unittest.mock import patch + +ROOT=Path(__file__).resolve().parents[1] +class PreparedImportTest(unittest.TestCase): + def setUp(self): + self.db=sqlite3.connect(':memory:'); self.db.row_factory=sqlite3.Row + for p in sorted((ROOT/'migrations').glob('*.sql')): self.db.executescript(p.read_text()) + self.base={'clientId':'c1','createdAt':'2026-09-01T00:00:00.000Z','updatedAt':'2026-09-01T00:00:00.000Z','deletedAt':None,'cryptoVersion':1,'isPinnedAside':False,'isPinnedHome':False} + self.data={'users':[{'_id':'u1','_creationTime':1790000000000,'email':'test@example.com'}],'authAccounts':[{'_id':'a1','userId':'u1','provider':'github','providerAccountId':'42'}],'userKeys':[{'userId':'u1','wrappedDek':'wrapped-test'}],'folders':[],'snippets':[{**self.base,'_id':'s1','ownerId':'u1','title':'ciphertext-test','code':'large-ciphertext-'*10000,'language':'plaintext','folderId':None}],'authSessions':[]} + self.writes=0; self.largest_sql=0 + def api(self,request,timeout): + body=json.loads(request.data); queries=body.get('batch',[body]); results=[] + with self.db: + for q in queries: + self.largest_sql=max(self.largest_sql,len(q['sql'])) + if not q['sql'].startswith('SELECT'): self.writes+=1 + c=self.db.execute(q['sql'],q.get('params',[])) + results.append({'success':True,'results':[dict(r) for r in c.fetchall()] if c.description else []}) + return io.BytesIO(json.dumps({'success':True,'result':results}).encode()) + def run_import(self): + with tempfile.TemporaryDirectory() as d: + p=Path(d)/'source.zip' + with zipfile.ZipFile(p,'w') as z: + for name,rows in self.data.items(): z.writestr(name+'/documents.jsonl','\n'.join(json.dumps(r) for r in rows)) + args=['importer',str(p),'--apply-remote','--source-frozen','--account-id','test-account','--database-id','test-db'] + with patch('sys.argv',args),patch.dict('os.environ',{'CLOUDFLARE_API_TOKEN':'test'}),patch('urllib.request.urlopen',self.api): + runpy.run_path(str(ROOT/'scripts/migrate-convex-to-d1.py'),run_name='__main__') + def test_large_ciphertext_bound_and_unchanged_edits_not_duplicated(self): + self.run_import() + self.assertEqual(json.loads(self.db.execute('SELECT data FROM snippets').fetchone()[0])['code'],self.data['snippets'][0]['code']) + revision=self.db.execute('SELECT revision FROM analytics_state').fetchone()[0] + self.run_import() + self.assertEqual(self.db.execute('SELECT revision FROM analytics_state').fetchone()[0],revision) + self.assertLess(self.largest_sql,1000) + def test_final_copy_reconciles_edits_and_deletions(self): + self.run_import() + self.data['snippets'][0]['updatedAt']='2026-09-02T00:00:00.000Z';self.data['snippets'][0]['code']='edited-ciphertext' + self.run_import() + self.assertEqual(json.loads(self.db.execute('SELECT data FROM snippets').fetchone()[0])['code'],'edited-ciphertext') + self.data['snippets']=[]; self.run_import() + self.assertEqual(self.db.execute('SELECT count(*) FROM snippets').fetchone()[0],0) + self.assertEqual(self.db.execute('SELECT count(*) FROM analytics_snippets').fetchone()[0],0) + def test_live_login_guard_runs_before_any_mutation(self): + self.db.execute("INSERT INTO sessions(id,sessionToken,userId,expires) VALUES('session','hash','u1','2026-10-01T00:00:00.000Z')") + with self.assertRaisesRegex(RuntimeError,'login sessions'): self.run_import() + self.assertEqual(self.writes,0) +if __name__=='__main__': unittest.main() diff --git a/src/__tests__/seed.test.ts b/src/__tests__/seed.test.ts index 0995393..44ada5c 100644 --- a/src/__tests__/seed.test.ts +++ b/src/__tests__/seed.test.ts @@ -19,10 +19,10 @@ function makeClient() { }; } -vi.mock("@/lib/convex", () => ({ - isConvexConfigured: () => mockClient !== null, - getConvexBrowserClient: () => mockClient, -})); +vi.mock("@/lib/cloud", () => ({cloud:{hasContent: async()=> { + if (!mockClient || !mockSignedIn) throw new Error("Not authenticated"); + return mockCloud.folders>0 || mockCloud.snippets>0; +}}})); import { db } from "@/lib/db"; import { readInitialWorkspace, seedWelcomeContent } from "@/lib/seed"; diff --git a/src/__tests__/sync.test.ts b/src/__tests__/sync.test.ts index b6210ee..0c61458 100644 --- a/src/__tests__/sync.test.ts +++ b/src/__tests__/sync.test.ts @@ -1,6 +1,5 @@ // fake-indexeddb/auto is loaded via vitest setupFiles (vitest.config.ts). import { describe, it, expect, beforeEach, vi } from "vitest"; -import { getFunctionName, type FunctionReference } from "convex/server"; import { assertNoFolderCycles, collectDescendantFolderIds } from "@convex/lib/hierarchy"; import type { CloudFolder, CloudSnippet, FolderRecord, SnippetRecord } from "@/lib/types"; @@ -147,36 +146,13 @@ function changes({ since }: { since: number | null }) { }; } -const convexClient = { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async query(reference: FunctionReference<"query">, args?: never): Promise { - switch (getFunctionName(reference)) { - case "workspace:changes": - return changes(args!); - case "workspace:list": - return { folders: ownedFolders().map(stripOwner), snippets: ownedSnippets().map(stripOwner) }; - case "workspace:hasContent": - return ownedFolders().length > 0 || ownedSnippets().length > 0; - default: - throw new Error(`Unexpected query ${getFunctionName(reference)}`); - } - }, - async mutation(reference: FunctionReference<"mutation">, args: never) { - switch (getFunctionName(reference)) { - case "workspace:push": - return push(args); - case "workspace:remove": - return remove(args); - default: - throw new Error(`Unexpected mutation ${getFunctionName(reference)}`); - } - }, +const cloudClient = { + changes: async (since: number | null) => changes({since}), + push: async (body: {folders: CloudFolder[]; snippets:CloudSnippet[]}) => push(body), + remove: async (body: {folderIds:string[];snippetIds:string[]}) => remove(body), + hasContent: async () => ownedFolders().length>0 || ownedSnippets().length>0, }; - -vi.mock("@/lib/convex", () => ({ - isConvexConfigured: () => true, - getConvexBrowserClient: () => convexClient, -})); +vi.mock("@/lib/cloud", () => ({cloud:{changes:(since:number|null)=>cloudClient.changes(since),push:(body:{folders:CloudFolder[];snippets:CloudSnippet[]})=>cloudClient.push(body),remove:(body:{folderIds:string[];snippetIds:string[]})=>cloudClient.remove(body),hasContent:()=>cloudClient.hasContent()}})); // ── Mocked encryption key ────────────────────────────────────────────────────── // `null` (the default) runs sync in plaintext mode, matching the pre-encryption @@ -443,6 +419,16 @@ describe("syncDirtyWorkspace() batching", () => { expect(order.indexOf(child.id)).toBeLessThan(order.indexOf(grandchild.id)); }); + it("bounds the combined folder/snippet payload, not just each list", async () => { + await db.folders.bulkAdd([makeFolder({dirty:true,name:'f'.repeat(1200000)}),makeFolder({dirty:true,name:'g'.repeat(1200000)})]); + await db.snippets.bulkAdd([makeSnippet({dirty:true,code:'a'.repeat(1200000)}),makeSnippet({dirty:true,code:'b'.repeat(1200000)})]); + const calls=vi.spyOn(cloudClient,'push'); + await syncDirtyWorkspace(USER); + expect(calls).toHaveBeenCalledTimes(2); + for (const [body] of calls.mock.calls) expect(new TextEncoder().encode(JSON.stringify(body)).length).toBeLessThanOrEqual(4000000); + calls.mockRestore(); + }); + it("fails clearly instead of sending an oversized record forever", async () => { const oversized = makeSnippet({ dirty: true, code: "x".repeat(4_100_000) }); await db.snippets.add(oversized); @@ -822,26 +808,8 @@ describe("fetchCloudWorkspace() incremental pull", () => { expect((await db.snippets.get(snippet.id))?.title).toBe("back"); }); - it("falls back to the full list when the backend has no incremental query yet", async () => { - const snippet = makeSnippet(); - pushFromElsewhere([], [cloudSnippet(snippet)]); - await db.syncCursors.put({ userId: USER, cursor: 1 }); - backendHasChanges = false; - - await fetchCloudWorkspace(USER); - - expect(await db.snippets.get(snippet.id)).toBeDefined(); - // Writes on that backend are not stamped, so the cursor must not be trusted. - expect(await db.syncCursors.get(USER)).toBeUndefined(); - - // Deletions still propagate, by absence, exactly as before. - cloud.snippets = []; - await fetchCloudWorkspace(USER); - expect(await db.snippets.get(snippet.id)).toBeUndefined(); - }); - it("does not turn other failures into a second, full read", async () => { - const failing = vi.spyOn(convexClient, "query").mockRejectedValueOnce(new Error("Not authenticated")); + const failing = vi.spyOn(cloudClient, "changes").mockRejectedValueOnce(new Error("Not authenticated")); await expect(fetchCloudWorkspace(USER)).rejects.toThrow("Not authenticated"); expect(failing).toHaveBeenCalledTimes(1); diff --git a/src/app/[locale]/layout.tsx b/src/app/[locale]/layout.tsx index e124c14..35bd212 100644 --- a/src/app/[locale]/layout.tsx +++ b/src/app/[locale]/layout.tsx @@ -1,5 +1,5 @@ import { Geist, Geist_Mono } from "next/font/google"; -import { ConvexClientProvider } from "@/components/ConvexClientProvider"; +import { CloudClientProvider } from "@/components/CloudClientProvider"; import { ServiceWorkerRegistration } from "@/components/ServiceWorkerRegistration"; import { ASIDE_WIDTH_INIT_SCRIPT } from "@/lib/asideWidth"; import { THEME_INIT_SCRIPT } from "@/lib/theme"; @@ -48,7 +48,7 @@ export default async function LocaleLayout({ {/* Same idea for the aside's stored width: applied before the panel paints so it never animates out from the default on every load. */}