From ed8311df657957040d2008389b6dcc32f90fee36 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:06:42 -0700 Subject: [PATCH 1/4] fix(backfill): skip messages whose pod is gone instead of failing (TASK-503) The prod --since run failed all 13 rows on pod_message_revisions_pod_id_fkey. syncPodFromMongo returns null when the pod is missing from Mongo too, so the retry hit the same FK. Such a row is an orphan of a deleted pod: it is now counted as `orphaned`, left untouched in Mongo, and never fails the run. `orphanedPending` reports the count before any write, dry run included. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- .../service/backfillLocalMessagesToPg.test.js | 20 ++++++++ .../scripts/backfillLocalMessagesToPg.test.js | 20 +++++++- .../scripts/backfill-local-messages-to-pg.ts | 48 +++++++++++++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/backend/__tests__/service/backfillLocalMessagesToPg.test.js b/backend/__tests__/service/backfillLocalMessagesToPg.test.js index d8955f1156..c6edb8ba88 100644 --- a/backend/__tests__/service/backfillLocalMessagesToPg.test.js +++ b/backend/__tests__/service/backfillLocalMessagesToPg.test.js @@ -140,6 +140,26 @@ d('TASK-503: local-upgrade backfill into PostgreSQL', () => { } }); + test('a message whose pod is gone from both stores is skipped as orphaned and never fails the run', async () => { + const deletedPod = new mongoose.Types.ObjectId(); + const orphan = await MongoMessage.create({ + podId: deletedPod, userId: USER, content: `${MARKER} deleted pod`, messageType: 'text', + }); + const live = await mongoMessage('live pod'); + const deps = productionDeps(pool); + + const dry = await backfillLocalMessages(deps, { force: true }); + expect(dry).toMatchObject({ dryRun: true, pending: 2, orphanedPending: 1, orphaned: 0 }); + + const applied = await backfillLocalMessages(deps, { apply: true, force: true }); + expect(applied).toMatchObject({ pending: 2, orphanedPending: 1, moved: 1, orphaned: 1, failed: 0 }); + expect(await pgCopies()).toEqual([live._id.toString()]); + const { rows } = await pool.query('SELECT id FROM pods WHERE id = $1', [deletedPod.toHexString()]); + expect(rows).toHaveLength(0); + const left = await MongoMessage.findById(orphan._id).lean(); + expect(left.reconciledAt).toBeUndefined(); + }); + test('since copies only Mongo rows created at or after it', async () => { const since = new Date('2026-09-30T00:00:00Z'); await mongoMessage('older history', new Date('2026-09-29T23:59:59Z')); diff --git a/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js index 66c15a0736..63f4f1dfff 100644 --- a/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js +++ b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js @@ -4,13 +4,14 @@ const { backfillLocalMessages, main, parseArgs } = require('../../../scripts/bac const doc = (id) => ({ _id: { toString: () => id }, podId: 'p', userId: 'u', content: id, createdAt: new Date('2026-01-01') }); -const fakeDeps = ({ pending = ['a1', 'a2', 'a3'], overlap = 0, copy } = {}) => { +const fakeDeps = ({ pending = ['a1', 'a2', 'a3'], overlap = 0, orphans = 0, copy } = {}) => { const copied = []; return { copied, countPending: jest.fn(async () => pending.length), newestPendingAt: jest.fn(async () => new Date('2026-01-01')), countOverlap: jest.fn(async () => overlap), + countOrphaned: jest.fn(async () => orphans), nextBatch: jest.fn(async (afterId, limit) => { const start = afterId ? pending.indexOf(afterId) + 1 : 0; return pending.slice(start, start + limit).map(doc); @@ -53,6 +54,22 @@ describe('backfillLocalMessages', () => { expect(report).toMatchObject({ refused: false, moved: 3 }); }); + it('a dry run reports how many pending rows belong to deleted pods', async () => { + const deps = fakeDeps({ orphans: 2 }); + const report = await backfillLocalMessages(deps); + expect(report).toMatchObject({ dryRun: true, pending: 3, orphanedPending: 2, orphaned: 0 }); + expect(deps.copy).not.toHaveBeenCalled(); + }); + + it('a row whose pod is gone is counted as orphaned, not failed, and the run continues', async () => { + const deps = fakeDeps({ + orphans: 1, + copy: async (d) => (d._id.toString() === 'a2' ? 'orphaned' : 'moved'), + }); + const report = await backfillLocalMessages(deps, { apply: true, batchSize: 1 }); + expect(report).toMatchObject({ moved: 2, orphaned: 1, alreadyPresent: 0, failed: 0 }); + }); + it('a failed row is counted and the run moves past it', async () => { const deps = fakeDeps({ copy: async (d) => { if (d._id.toString() === 'a2') throw new Error('bad'); return 'moved'; } }); const report = await backfillLocalMessages(deps, { apply: true, batchSize: 1 }); @@ -88,6 +105,7 @@ describe('main() exit code', () => { expect((await run(fakeDeps())).exitCode).toBe(0); expect((await run(fakeDeps(), ['node', 'script'])).exitCode).toBe(0); expect((await run(fakeDeps({ overlap: 4 }), ['node', 'script'])).exitCode).toBe(2); + expect((await run(fakeDeps({ orphans: 3, copy: async () => 'orphaned' }))).exitCode).toBe(0); }); }); diff --git a/backend/scripts/backfill-local-messages-to-pg.ts b/backend/scripts/backfill-local-messages-to-pg.ts index 993875dbdb..697c53edf3 100644 --- a/backend/scripts/backfill-local-messages-to-pg.ts +++ b/backend/scripts/backfill-local-messages-to-pg.ts @@ -36,6 +36,12 @@ * npm run backfill:local-messages -- --apply * npm run backfill:local-messages -- --since 2026-09-30T00:00:00Z --apply --force * + * A row whose pod no longer exists in MongoDB is an orphan: its pod was + * deleted, so there is no pod to mirror into PG and the FK on `pods` refuses + * the copy. Those rows are skipped, left untouched in Mongo and counted as + * `orphaned`; they never fail the run. `orphanedPending` reports how many of + * the pending rows are orphans before anything is written, dry run included. + * * Exit codes: 0 done, or a dry run that would proceed; 1 some rows failed; * 2 refused (a dry run that finds the overlap refuses too, so it also exits 2). */ @@ -44,7 +50,7 @@ import mongoose from 'mongoose'; import { copyOne, ensureLegacyIdIndex } from '../services/pgMessageReconcileService'; -type CopyResult = 'moved' | 'present'; +type CopyResult = 'moved' | 'present' | 'orphaned'; export interface PendingDoc { _id: { toString(): string }; @@ -59,6 +65,8 @@ export interface LocalBackfillDeps { newestPendingAt: () => Promise; /** PG rows with no legacy_mongo_id written at or before `at`. */ countOverlap: (at: Date) => Promise; + /** Pending rows whose pod no longer exists in MongoDB. */ + countOrphaned: () => Promise; /** The next pending docs after `afterId`, oldest id first. */ nextBatch: (afterId: string | null, limit: number) => Promise; copy: (doc: PendingDoc) => Promise; @@ -71,6 +79,10 @@ export interface LocalBackfillReport { refused: boolean; moved: number; alreadyPresent: number; + /** Pending rows whose pod is gone, counted before any write. */ + orphanedPending: number; + /** Rows this run skipped because their pod is gone. */ + orphaned: number; failed: number; } @@ -81,8 +93,17 @@ export async function backfillLocalMessages( const pending = await deps.countPending(); const newest = pending ? await deps.newestPendingAt() : null; const overlap = newest ? await deps.countOverlap(newest) : 0; + const orphanedPending = pending ? await deps.countOrphaned() : 0; const report: LocalBackfillReport = { - dryRun: !apply, pending, overlap, refused: overlap > 0 && !force, moved: 0, alreadyPresent: 0, failed: 0, + dryRun: !apply, + pending, + overlap, + refused: overlap > 0 && !force, + moved: 0, + alreadyPresent: 0, + orphanedPending, + orphaned: 0, + failed: 0, }; if (!apply || report.refused || !pending) return report; @@ -92,7 +113,9 @@ export async function backfillLocalMessages( if (!batch.length) break; for (const doc of batch) { try { - if (await deps.copy(doc) === 'moved') report.moved += 1; + const result = await deps.copy(doc); + if (result === 'moved') report.moved += 1; + else if (result === 'orphaned') report.orphaned += 1; else report.alreadyPresent += 1; } catch (err) { report.failed += 1; @@ -116,6 +139,8 @@ export const productionDeps = (pool: PgPool, { since }: BackfillScope = {}): Loc const Message = require('../models/Message'); // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require const { syncPodFromMongo } = require('../services/pgPodSyncService'); + // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require + const MongoPod = require('../models/Pod'); const unreconciled = since ? { reconciledAt: { $exists: false }, createdAt: { $gte: since } } : { reconciledAt: { $exists: false } }; @@ -132,13 +157,22 @@ export const productionDeps = (pool: PgPool, { since }: BackfillScope = {}): Loc ); return Number(rows[0]?.n || 0); }, + countOrphaned: async () => { + const podIds = await Message.distinct('podId', unreconciled); + if (!podIds.length) return 0; + const live = await MongoPod.find({ _id: { $in: podIds } }).distinct('_id'); + return Message.countDocuments({ ...unreconciled, podId: { $nin: live } }); + }, nextBatch: (afterId, limit) => Message.find(afterId ? { ...unreconciled, _id: { $gt: new mongoose.Types.ObjectId(afterId) } } : unreconciled).sort({ _id: 1 }).limit(limit).lean(), - // Same pod fallback as the reconciler: a Mongo-only pod is mirrored, then retried. + // Same pod fallback as the reconciler: a Mongo-only pod is mirrored, then + // retried. syncPodFromMongo returns null when the pod is gone from Mongo + // too; retrying would hit the same FK, so the row is an orphan. copy: (doc) => copyOne(doc).catch(async (err: { code?: string }) => { if (err.code !== '23503') throw err; - await syncPodFromMongo(doc.podId.toString(), doc.userId.toString()); + const pod = await syncPodFromMongo(doc.podId.toString(), doc.userId.toString()); + if (!pod) return 'orphaned' as const; return copyOne(doc); }), }; @@ -229,6 +263,10 @@ export async function main( } else if (!apply) { console.log('DRY RUN — nothing written. Re-run with --apply.'); } + if (report.orphanedPending) { + console.log(`${report.orphanedPending} pending message(s) belong to pods that no longer exist in MongoDB;` + + ' they are skipped and left in MongoDB.'); + } process.exitCode = exitCodeFor(report); return process.exitCode; } finally { From 8b6c9a45d70cf4bb7511fb5e2cf0034fdb052e0a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:20:17 -0700 Subject: [PATCH 2/4] fix(backfill): an orphan's pod is missing from both stores (TASK-503) countOrphaned checked Mongo only, so a pod that lives only in PG (it copies without a mirror) was miscounted as an orphan. The real-DB test caught it: the dry run reported orphanedPending 2 for one live PG pod and one deleted pod. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- .../scripts/backfill-local-messages-to-pg.ts | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/backend/scripts/backfill-local-messages-to-pg.ts b/backend/scripts/backfill-local-messages-to-pg.ts index 697c53edf3..bc4072a6e0 100644 --- a/backend/scripts/backfill-local-messages-to-pg.ts +++ b/backend/scripts/backfill-local-messages-to-pg.ts @@ -36,11 +36,11 @@ * npm run backfill:local-messages -- --apply * npm run backfill:local-messages -- --since 2026-09-30T00:00:00Z --apply --force * - * A row whose pod no longer exists in MongoDB is an orphan: its pod was - * deleted, so there is no pod to mirror into PG and the FK on `pods` refuses - * the copy. Those rows are skipped, left untouched in Mongo and counted as - * `orphaned`; they never fail the run. `orphanedPending` reports how many of - * the pending rows are orphans before anything is written, dry run included. + * A row whose pod is in neither store is an orphan: its pod was deleted, so + * there is nothing to mirror into PG and the FK on `pods` refuses the copy. + * Those rows are skipped, left untouched in Mongo and counted as `orphaned`; + * they never fail the run. `orphanedPending` reports how many of the pending + * rows are orphans before anything is written, dry run included. * * Exit codes: 0 done, or a dry run that would proceed; 1 some rows failed; * 2 refused (a dry run that finds the overlap refuses too, so it also exits 2). @@ -157,11 +157,17 @@ export const productionDeps = (pool: PgPool, { since }: BackfillScope = {}): Loc ); return Number(rows[0]?.n || 0); }, + // An orphan's pod is in neither store: a pod already in PG copies without + // a mirror, and a Mongo-only pod gets mirrored, so only the rest fail the FK. countOrphaned: async () => { - const podIds = await Message.distinct('podId', unreconciled); + const podIds: Array<{ toString(): string }> = await Message.distinct('podId', unreconciled); if (!podIds.length) return 0; - const live = await MongoPod.find({ _id: { $in: podIds } }).distinct('_id'); - return Message.countDocuments({ ...unreconciled, podId: { $nin: live } }); + const ids = podIds.map(String); + const inMongo: Array<{ toString(): string }> = await MongoPod.find({ _id: { $in: podIds } }).distinct('_id'); + const { rows } = await pool.query('SELECT id FROM pods WHERE id = ANY($1)', [ids]); + const live = new Set([...inMongo.map(String), ...rows.map((r) => String(r.id))]); + const gone = podIds.filter((id) => !live.has(String(id))); + return gone.length ? Message.countDocuments({ ...unreconciled, podId: { $in: gone } }) : 0; }, nextBatch: (afterId, limit) => Message.find(afterId ? { ...unreconciled, _id: { $gt: new mongoose.Types.ObjectId(afterId) } } From c036cdc1fe37494688cb0526e78cb42b92369425 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:53:07 -0700 Subject: [PATCH 3/4] docs(backfill): orphan wording names both stores (TASK-503) Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- backend/scripts/backfill-local-messages-to-pg.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/scripts/backfill-local-messages-to-pg.ts b/backend/scripts/backfill-local-messages-to-pg.ts index bc4072a6e0..e3ea5ed555 100644 --- a/backend/scripts/backfill-local-messages-to-pg.ts +++ b/backend/scripts/backfill-local-messages-to-pg.ts @@ -65,7 +65,7 @@ export interface LocalBackfillDeps { newestPendingAt: () => Promise; /** PG rows with no legacy_mongo_id written at or before `at`. */ countOverlap: (at: Date) => Promise; - /** Pending rows whose pod no longer exists in MongoDB. */ + /** Pending rows whose pod exists in neither MongoDB nor PG `pods`. */ countOrphaned: () => Promise; /** The next pending docs after `afterId`, oldest id first. */ nextBatch: (afterId: string | null, limit: number) => Promise; @@ -270,7 +270,7 @@ export async function main( console.log('DRY RUN — nothing written. Re-run with --apply.'); } if (report.orphanedPending) { - console.log(`${report.orphanedPending} pending message(s) belong to pods that no longer exist in MongoDB;` + console.log(`${report.orphanedPending} pending message(s) belong to pods that exist in neither MongoDB nor PG;` + ' they are skipped and left in MongoDB.'); } process.exitCode = exitCodeFor(report); From 06f328422a2094b78955f0fe464275077ee6964b Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 13:55:49 -0700 Subject: [PATCH 4/4] test(backfill): a Mongo-only pod is never counted as an orphan (TASK-503) Pins the Mongo half of countOrphaned: the mirror test now asserts orphanedPending 0 on the dry run and orphaned 0 on apply. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- backend/__tests__/service/backfillLocalMessagesToPg.test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/backend/__tests__/service/backfillLocalMessagesToPg.test.js b/backend/__tests__/service/backfillLocalMessagesToPg.test.js index c6edb8ba88..a7c3182818 100644 --- a/backend/__tests__/service/backfillLocalMessagesToPg.test.js +++ b/backend/__tests__/service/backfillLocalMessagesToPg.test.js @@ -129,8 +129,11 @@ d('TASK-503: local-upgrade backfill into PostgreSQL', () => { podId: mongoOnlyPod, userId: USER, content: `${MARKER} mongo-only pod`, messageType: 'text', }); try { + // A live Mongo pod is not an orphan, before the mirror and after it. + const dry = await backfillLocalMessages(productionDeps(pool), { force: true }); + expect(dry).toMatchObject({ dryRun: true, pending: 1, orphanedPending: 0 }); const report = await backfillLocalMessages(productionDeps(pool), { apply: true, force: true }); - expect(report).toMatchObject({ pending: 1, moved: 1, failed: 0 }); + expect(report).toMatchObject({ pending: 1, moved: 1, failed: 0, orphanedPending: 0, orphaned: 0 }); expect(await pgCopies()).toEqual([doc._id.toString()]); const { rows } = await pool.query('SELECT id FROM pods WHERE id = $1', [mongoOnlyPod.toHexString()]); expect(rows).toHaveLength(1);