✨ feat: migrate KlipCode backend and GitHub sessions to D1

.dev.vars.example

@@ -1,10 +1,7 @@
-# Wrangler does not read .env, so `pnpm preview` needs the same values here.
-# Copy to .dev.vars and fill in.
-#
-# The Convex deployment the previewed build talks to. `pnpm dev:backend` puts
-# this in .env.local; copy the same value here.
-NEXT_PUBLIC_CONVEX_URL=
-
-# See .env.example — same key, same warning: back it up, and never set it as a
-# Convex environment variable.
+# Copy to .dev.vars. Never commit real credentials or the existing master key.
+AUTH_URL=http://localhost:3000
+AUTH_SECRET=
+AUTH_GITHUB_ID=
+AUTH_GITHUB_SECRET=
+# Base64-encoded 32-byte key. Production must keep its existing value forever.
 ENCRYPTION_MASTER_KEY=

.env.example

@@ -1,18 +1,3 @@
-# Copy to .env for local development with `pnpm dev` (Node runtime).
-# For `pnpm preview` / `pnpm deploy` (Cloudflare Workers runtime), the same
-# values must live in .dev.vars instead — wrangler does not read .env.
-# In production, set them as Worker secrets: wrangler secret put <NAME>
-#
-# NEXT_PUBLIC_CONVEX_URL is NOT listed here: `pnpm dev:backend` writes it (and
-# CONVEX_DEPLOYMENT) into .env.local for you. Without a deployment URL, sign-in
-# and cloud storage are unavailable; only the device-bound guest workspace is
-# available.
-
-# Master key (KEK) that wraps every per-user data-encryption key. It must be
-# 32 random bytes in base64 — generate one with: openssl rand -base64 32
-# Deliberately kept on Cloudflare and NEVER set as a Convex environment
-# variable: that separation is what stops the encrypted records and the key
-# that opens them from living with the same provider.
-# Losing it makes all encrypted cloud records unrecoverable: back it up in a
-# password manager. If it is unset, the app syncs in plaintext (cryptoVersion 0).
-ENCRYPTION_MASTER_KEY=
+# Runtime configuration belongs in .dev.vars; see .dev.vars.example.
+# Keep remote Images/AI disabled when testing without Cloudflare credentials.
+KLIPCODE_REMOTE_BINDINGS=false

.env.production

@@ -1,21 +1,2 @@
-# Production build values. Committed on purpose.
-#
-# NEXT_PUBLIC_* is inlined into the client bundle at BUILD time, so it has to be
-# present when the build runs — setting it as a Worker runtime variable does
-# nothing, the bundle is already compiled by then. Cloudflare Workers Builds
-# runs `opennextjs-cloudflare build` on its own runners, where neither
-# .env.local nor the `convex deploy --cmd` wrapper in `pnpm deploy` exists, so
-# without this file the production build compiles with no deployment URL and the
-# app silently falls back to a device-only guest workspace.
-#
-# Keeping it here rather than in the Cloudflare dashboard means the value is
-# reviewable, versioned, and cannot drift out of sync with the deployment.
-#
-# This is not a secret: the URL ships inside the client bundle either way, and
-# access is gated by the identity checks inside each Convex function, never by
-# the URL being unknown. The actual secret (ENCRYPTION_MASTER_KEY) stays a
-# Cloudflare Worker secret and is never committed.
-#
-# .env.local wins over this file, so local builds keep pointing at the dev
-# deployment; only builds without a .env.local (i.e. CI/Cloudflare) use it.
-NEXT_PUBLIC_CONVEX_URL=https://befitting-badger-749.convex.cloud
+# The cloud backend is same-origin and configured through runtime D1 bindings.
+# No deployment URL or credentials are embedded in the browser bundle.

.github/workflows/ci.yml

@@ -29,6 +29,9 @@ jobs:
       - name: Test
         run: pnpm test
 
+      - name: Test migration importer
+        run: python3 scripts/test-migrate-convex-to-d1.py
+
       - name: Build
         run: pnpm build
 

README.md

@@ -16,20 +16,21 @@ guest workspace that stays on the device you are using.
 - Write and preview Markdown documents in a rich TipTap editor.
 - Generate a name for an untitled snippet with Workers AI (Llama 3.2 3B
   Instruct) when signed in.
-- Sync the library through Convex after GitHub sign-in. Cloud records are
-  encrypted per user when the Cloudflare Worker secret `ENCRYPTION_MASTER_KEY`
-  is set; without it, records are synced in plaintext (`cryptoVersion 0`).
+- Sync the library through Cloudflare D1 after GitHub sign-in. Cloud records
+  are encrypted per user using the existing `ENCRYPTION_MASTER_KEY` Worker
+  secret. Missing keys fail closed rather than uploading plaintext.
   A device downloads the whole library once, then only what changed since its
-  last sync (a server-clock cursor), which keeps Convex database I/O small.
+  last sync (a server-clock cursor), which keeps D1 database I/O small.
 - Use the app in English or Spanish, in light or dark theme, with keyboard
   shortcuts and a search palette. A service worker caches the app shell so the
   UI still loads offline.
 
 ## Tech stack
 
 Next.js 16 (App Router, React 19) and Tailwind CSS 4, deployed as a Cloudflare
-Worker through OpenNext. Convex provides the backend, the database, and GitHub
-authentication; local snippet storage uses IndexedDB via Dexie.
+Worker through OpenNext. D1 stores the workspace and Auth.js database sessions;
+GitHub provides authentication. Local snippet storage uses IndexedDB via Dexie.
+The private analytics console uses the same D1 database without a VPS.
 
 ## Requirements
 
@@ -47,23 +48,25 @@ pnpm dev
 
 Open <http://localhost:3000>. This is enough to work on the guest workspace.
 
-To develop cloud sync and authentication, run the Convex development backend in
-a second terminal before starting the app:
+To develop cloud sync, copy [.dev.vars.example](.dev.vars.example) to
+`.dev.vars`, fill in a development OAuth app and encryption key, then run:
 
 ```bash
 pnpm dev:backend
 pnpm dev
 ```
 
-`pnpm dev:backend` writes the development deployment values to `.env.local`.
-For encrypted cloud records, copy [.env.example](.env.example) to `.env` and set
-`ENCRYPTION_MASTER_KEY`. GitHub sign-in also requires a GitHub OAuth app and
-these Convex environment variables:
+`pnpm dev:backend` applies D1 migrations locally. The GitHub callback is
+`http://localhost:3000/api/auth/callback/github` for development and
+`https://klipcode.com/api/auth/callback/github` for production. Use separate
+OAuth applications for these environments. Missing OAuth credentials leave
+the guest workspace available.
 
-```bash
-pnpm exec convex env set AUTH_GITHUB_ID <client-id>
-pnpm exec convex env set AUTH_GITHUB_SECRET <client-secret>
-```
+Production uses `DB` as a D1 binding and Worker secrets `AUTH_SECRET`,
+`AUTH_GITHUB_ID`, `AUTH_GITHUB_SECRET`, `AUTH_URL` and
+`ENCRYPTION_MASTER_KEY`. Never replace the production master key: existing
+wrapped keys and ciphertext depend on it. `pnpm deploy` applies versioned D1
+migrations before building and deploying the Worker.
 
 ## Checks and scripts
 
@@ -82,7 +85,8 @@ CI runs all of the above on every push to `main` and on every pull request.
 ## Further reading
 
 - [Engineering audit](docs/audit/engineering-audit.md)
-- [Convex backend](convex/)
+- [D1 backend](src/server/)
+- [Migration and rollback](docs/d1-migration.md)
 - [Environment example](.env.example)
 
 KlipCode is released under the [MIT License](LICENSE).

convex.json

@@ -0,0 +1,3 @@
+{
+  "functions": "legacy/convex"
+}

docs/audit/engineering-audit.md

@@ -1,5 +1,8 @@
 # Engineering Audit
 
+> Historical audit of the Convex architecture. The current D1 architecture and
+> provider-separation change are documented in [D1 migration](../d1-migration.md).
+
 **Audit date:** 2026-08-07  
 **Scope:** application code, Convex functions, local persistence, cloud sync, authentication, API routes, dependency graph, accessibility-sensitive UI paths, and a production-style browser smoke test.
 

docs/d1-migration.md

@@ -0,0 +1,110 @@
+# D1 migration
+
+The application and analytics console use Cloudflare Workers and the `klipcode`
+D1 database. Production request paths have no Convex calls or VPS dependency.
+GitHub accounts retain their existing user ids, so IndexedDB ownership and R2
+image paths remain valid. Ciphertext and wrapped DEKs are copied without
+re-encryption. The master key stays the existing `klipcode` Worker secret.
+
+## Data and sessions
+
+- D1 workspace reads and writes include the authenticated owner in every query.
+- Browser writes require the app's origin and expected account id; a cookie
+  change during sync cannot upload ciphertext into another account.
+- Auth.js issues HTTP-only, Secure, SameSite cookies. Only hashes of opaque
+  session tokens are stored in D1. GitHub numeric identity ids resolve to the
+  imported users. Existing Convex sessions require GitHub sign-in again.
+- Writes use indexed primary keys, last-write-wins timestamps and atomic D1
+  batches. SQL triggers maintain deletion logs and analytics metadata.
+- Pulls use a server-clock cursor, a one-minute overlap and a 30-day deletion
+  window. Old cursors and the pre-migration backend epoch force a full pull.
+- Encryption failures stop sync and retain dirty local records for retry.
+- D1 metadata and Worker secrets share a provider. Database exports still cannot
+  decrypt records alone, but this no longer provides separation of providers.
+
+## Repeatable copy
+
+Keep all exports and generated SQL outside Git, in a protected directory on
+persistent storage or an external backup. Both contain private account data.
+First test against local D1, then the remote database before routing app traffic.
+
+```bash
+pnpm exec convex export --prod --path /protected/convex.zip
+python3 scripts/migrate-convex-to-d1.py /protected/convex.zip \
+  --analytics /protected/old-analytics-snapshot.json --sql /protected/import.sql
+pnpm exec wrangler d1 migrations apply DB --remote
+pnpm exec wrangler d1 execute DB --remote --file /protected/import.sql
+python3 scripts/migrate-convex-to-d1.py /protected/convex.zip --verify
+```
+
+For the final frozen copy, use prepared REST batches rather than another bulk
+SQL import. Cloudflare returned `D1_RESET_DO` on the large final SQL file during
+the production migration. Prepared parameters avoid oversized SQL statements;
+unchanged workspace records are skipped to preserve edit history and reduce
+writes. The script refuses this path if D1 already contains login sessions.
+Use a D1 API token from the environment or an existing protected Wrangler OAuth
+configuration; never put a token in a command argument.
+The copy script requires Python 3.11 or newer. Run its isolated SQLite/API tests
+with `python3 scripts/test-migrate-convex-to-d1.py`.
+
+```bash
+python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip \
+  --apply-remote --source-frozen \
+  --account-id YOUR_ACCOUNT_ID --database-id YOUR_DATABASE_ID \
+  --oauth-config /protected/wrangler/config/default.toml
+python3 scripts/migrate-convex-to-d1.py /protected/convex-final.zip --verify
+```
+
+The importer verifies every user id, provider identity, ciphertext field and
+wrapped key. It is idempotent only **before cutover**: it also reconciles source
+deletions. Never re-import over a live D1 workspace.
+
+Before the final copy, freeze the archived Convex workspace using its
+`KLIPCODE_MIGRATED=true` environment guard. A stale browser then keeps its
+unsynced changes locally until it reloads and signs in to the D1 version.
+Disable the old VPS deploy timer and protect the production Worker from old
+automatic deploy commands until the migration branch is published. Add the
+new GitHub return URL before deploying; remove the old URL after validation.
+
+## Rollback
+
+Before D1 accepts user writes, restore the recorded pre-migration Worker
+version, old GitHub callback and unset `KLIPCODE_MIGRATED` in Convex. Source data
+is preserved. After D1 accepts writes, **do not switch back directly**: freeze D1
+writes, export it, reconcile new records/deletions into the source, and verify
+before restoring traffic. Worker rollback alone cannot roll database state back.
+
+Convex code remains archived solely for the source freeze, verification and
+rollback. It is not part of the deployed app backend. Remove the legacy project
+and credentials only after the backup retention window and explicit approval.
+
+## Production cutover on 2026-09-30
+
+The verified final export contains 14 users, 14 GitHub identities, 13 wrapped
+keys, 43 folders and 302 snippets. App version
+`6682cee2-f690-4748-a1d8-957e1fcbc996` and console version
+`bf260137-d3d4-483c-8f59-934723ad4a04` completed the cutover. Production checks
+confirmed GitHub sign-in resolves to an imported user, existing encrypted fields
+decrypt, and encrypted create/edit/delete succeeds. The old GitHub callback was
+removed and the Convex source remains frozen for recovery.
+
+The production Cloudflare build command currently uploads a version without
+activating traffic. Once this migration is published to the production branch,
+restore the deploy command to `pnpm deploy`, which applies D1 migrations before
+building and deploying. Do not restore automatic activation while the branch
+still contains the old Convex application. The VPS origin, tunnel and deploy
+timer are disabled; infrastructure provisioning also keeps them disabled.
+
+The private console polls a single revision row every ten seconds while visible,
+then loads cached metadata when the revision changes. It never reads ciphertext
+for analytics and makes no Convex requests.
+
+## Legacy timestamp compatibility
+
+Imported records retain ISO timestamps with explicit offsets such as
+`2026-07-16T11:04:39.716+00:00`, including variable fractional precision. The
+backend accepts those alongside `Z` timestamps, preserves their original values,
+and compares update timestamps by instant for last-write-wins. Version
+`1f3b6e7d-69df-4ee1-b2a0-d08ba7bdabbf` fixes the initial validation regression
+that rejected edits to these records. A real D1 regression test covers the
+imported format and stale writes with a different UTC offset.

e2e/mobile.spec.ts

@@ -78,7 +78,9 @@ test("a swipe that falls short springs back", async ({ page }) => {
 test("a short flick still commits", async ({ page }) => {
   await gotoMobileApp(page);
 
-  await dragHorizontally(page, panel(page), { dx: -60 });
+  // One native move keeps the flick fast even on a loaded CI/VPS host; eight
+  // separate protocol round trips can accidentally turn it into a slow drag.
+  await dragHorizontally(page, panel(page), { dx: -60, steps: 1 });
   await expectSelected(page, "My Space");
 });
 

eslint.config.mjs

@@ -21,7 +21,7 @@ const eslintConfig = defineConfig([
     // Isolated remote dev output (generated by KLIPCODE_DIST_DIR).
     ".next-remote/**",
     // Auto-generated by `npx convex dev`; not meant to be linted.
-    "convex/_generated/**",
+    "legacy/convex/_generated/**",
   ]),
 ]);
 

legacy/README.md

@@ -0,0 +1,10 @@
+# Legacy source backend
+
+This Convex project is retained only for migration verification, the source
+write freeze and rollback. No deployed application route imports it. Production
+is `src/server` and `migrations`, using Workers bindings and D1.
+
+`KLIPCODE_MIGRATED=true` blocks legacy workspace reads and writes. Unset it only
+when rolling back before D1 accepts new user writes, or after reconciling D1.
+The CLI functions directory is configured in the root `convex.json` solely to
+keep recovery commands repeatable. Convex packages are development dependencies.

legacy/convex/auth.ts

@@ -0,0 +1,37 @@
+import GitHub from "@auth/core/providers/github";
+import type { ActionCtx } from "./_generated/server";
+import { action } from "./_generated/server";
+import { v } from "convex/values";
+import { convexAuth } from "@convex-dev/auth/server";
+
+// GitHub is the only provider, matching the single "Sign in with GitHub" button
+// in the aside. Credentials come from the Convex deployment env
+// (AUTH_GITHUB_ID / AUTH_GITHUB_SECRET), never from the client bundle.
+const legacy = convexAuth({
+  providers: [GitHub],
+});
+
+export const { auth, signOut, store, isAuthenticated } = legacy;
+// Stop stale tabs from refreshing legacy tokens after the final copy.
+export const signIn = action({
+  args: {
+    provider: v.optional(v.string()),
+    params: v.optional(v.any()),
+    verifier: v.optional(v.string()),
+    refreshToken: v.optional(v.string()),
+    calledBy: v.optional(v.string()),
+  },
+  handler: async (ctx, args) => {
+    if (process.env.KLIPCODE_MIGRATED === "true")
+      throw new Error(
+        "KlipCode has moved. Reload and sign in at klipcode.com.",
+      );
+    const source = legacy.signIn as unknown as {
+      _handler: (
+        ctx: ActionCtx,
+        args: Record<string, unknown>,
+      ) => Promise<unknown>;
+    };
+    return source._handler(ctx, args);
+  },
+});

convex/http.ts → legacy/convex/http.ts

@@ -5,6 +5,6 @@ import { auth } from "./auth";
 const http = httpRouter();
 
 // Mounts the OAuth callback routes (/api/auth/*) on the deployment's .site domain.
-auth.addHttpRoutes(http);
+if (process.env.KLIPCODE_MIGRATED !== "true") auth.addHttpRoutes(http);
 
 export default http;

convex/lib/auth.ts → legacy/convex/lib/auth.ts

@@ -13,6 +13,9 @@ type AuthCtx = { auth: { getUserIdentity: () => Promise<{ subject: string } | nu
  * only the first segment identifies the account.
  */
 export async function requireUserId(ctx: AuthCtx): Promise<Id<"users">> {
+  if (process.env.KLIPCODE_MIGRATED === "true") {
+    throw new Error("KlipCode has moved. Reload the app and sign in again; unsynced changes remain on this device.");
+  }
   const identity = await ctx.auth.getUserIdentity();
 
   if (!identity) {