From dfbb745d77eb5119fa4116d0c64b4dd2893a981a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 06:31:16 -0700 Subject: [PATCH 1/4] refactor(messages): remove the Mongo read/delete fallbacks and the reconciler (TASK-503) With PG_HOST set, PostgreSQL is now the only message store for reads and deletes too, matching the writes from #2989. A PG failure is an error or a retryable 503 (MESSAGE_STORE_UNAVAILABLE), never a silent MongoDB read. - repositories/messageRepository: PG only with PG_HOST, Mongo only without. - agentMessageService.getRecentMessages and authorIdsSince: same gate; the assigned-task wake now fails closed on a PG failure instead of reading Mongo. - deleteMessage: PG only with PG_HOST; PG unreachable is a 503 that touches neither store. The best-effort removal of a copied row's legacy Mongo copy stays, so the Mongo-only readers stop counting it. - pgMessageReconcileService and its boot cron are removed. copyOne and ensureLegacyIdIndex move unchanged to services/legacyMessageCopy for the backfill script; their real-DB tests move with them. Merge only after the operator backfill (--since 2026-09-30) has copied the 13 recent Mongo rows: the reconciler never moved the 9 unflagged ones. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/high Agent-Model: claude/claude-opus-5-5/medium --- .../service/backfillLocalMessagesToPg.test.js | 2 +- ...cile.test.js => legacyMessageCopy.test.js} | 151 ++--------- backend/__tests__/service/messages.test.js | 5 + .../controllers/messageController.test.js | 40 ++- .../repositories/messageRepository.test.js | 26 +- .../scripts/backfillLocalMessagesToPg.test.js | 2 +- backend/__tests__/unit/server.test.js | 7 - ...agentMessageService.authorIdsSince.test.js | 37 +-- .../agentMessageService.threadReads.test.js | 19 +- .../nativeRuntimeService.hostedBroker.test.js | 3 + ...ativeRuntimeService.recentMessages.test.js | 8 +- .../nativeRuntimeService.webFetch.test.js | 9 +- backend/controllers/messageController.ts | 37 ++- backend/models/Message.ts | 11 +- backend/repositories/messageRepository.ts | 19 +- .../scripts/backfill-local-messages-to-pg.ts | 10 +- backend/server.ts | 2 - backend/services/agentMessageService.ts | 143 +++++------ backend/services/legacyMessageCopy.ts | 124 +++++++++ backend/services/pgMessageReconcileService.ts | 240 ------------------ docs/architecture/long-term-review-2026-10.md | 4 +- docs/plans/pod-playbooks.md | 2 +- 22 files changed, 356 insertions(+), 545 deletions(-) rename backend/__tests__/service/{pgMessageReconcile.test.js => legacyMessageCopy.test.js} (56%) create mode 100644 backend/services/legacyMessageCopy.ts delete mode 100644 backend/services/pgMessageReconcileService.ts diff --git a/backend/__tests__/service/backfillLocalMessagesToPg.test.js b/backend/__tests__/service/backfillLocalMessagesToPg.test.js index a7c3182818..312fab5e9a 100644 --- a/backend/__tests__/service/backfillLocalMessagesToPg.test.js +++ b/backend/__tests__/service/backfillLocalMessagesToPg.test.js @@ -64,7 +64,7 @@ d('TASK-503: local-upgrade backfill into PostgreSQL', () => { // eslint-disable-next-line global-require ({ backfillLocalMessages, productionDeps } = require('../../scripts/backfill-local-messages-to-pg')); // eslint-disable-next-line global-require - ({ ensureLegacyIdIndex } = require('../../services/pgMessageReconcileService')); + ({ ensureLegacyIdIndex } = require('../../services/legacyMessageCopy')); expect(await ensureLegacyIdIndex()).toBeTruthy(); }); diff --git a/backend/__tests__/service/pgMessageReconcile.test.js b/backend/__tests__/service/legacyMessageCopy.test.js similarity index 56% rename from backend/__tests__/service/pgMessageReconcile.test.js rename to backend/__tests__/service/legacyMessageCopy.test.js index a731be09ff..c54db2475b 100644 --- a/backend/__tests__/service/pgMessageReconcile.test.js +++ b/backend/__tests__/service/legacyMessageCopy.test.js @@ -1,10 +1,12 @@ /** - * TASK-313 part 2 โ€” Mongo fallback messages reconciled into PostgreSQL. + * TASK-503 โ€” a legacy MongoDB message copied into PostgreSQL by `copyOne`. * * Runs against real PostgreSQL and real MongoDB (Tier 1). The claims need a * database that enforces them: ON CONFLICT on a unique index, a foreign key - * from messages.pod_id to pods(id), and two concurrent passes racing on the - * same rows. pg-mem can do none of that faithfully. + * from messages.pod_id to pods(id), and two concurrent copies racing on the + * same row. pg-mem can do none of that faithfully. The cases that tested the + * removed background reconciler's pass (batching, deferral, poison rows) went + * with it; the backfill script's own suite covers the operator run. */ const fs = require('fs'); const path = require('path'); @@ -18,7 +20,6 @@ const MARKER = 'tier1-reconcile-313'; let pool; let MongoMessage; -let reconcile; let copyOne; let ensureLegacyIdIndex; let PGMessage; @@ -51,7 +52,7 @@ const pgRowsFor = async (mongoId) => (await pool.query( [String(mongoId)], )).rows; -d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { +d('TASK-503: copy a legacy Mongo message into PostgreSQL', () => { beforeAll(async () => { pool = connect(); await pool.query(fs.readFileSync(path.join(__dirname, '../../config/schema.sql'), 'utf8')); @@ -65,11 +66,12 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { // eslint-disable-next-line global-require MongoMessage = require('../../models/Message'); // eslint-disable-next-line global-require - ({ reconcileMongoFallbackMessages: reconcile, copyOne, ensureLegacyIdIndex } = require('../../services/pgMessageReconcileService')); + ({ copyOne, ensureLegacyIdIndex } = require('../../services/legacyMessageCopy')); // eslint-disable-next-line global-require PGMessage = require('../../models/pg/Message'); // eslint-disable-next-line global-require MessageReaction = require('../../models/pg/MessageReaction').default; + await ensureLegacyIdIndex(); }); beforeEach(async () => { @@ -91,9 +93,7 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { test('moves a fallback message once and keeps its Mongo id as the legacy id', async () => { const doc = await insertFallback('a'); - const pass = await reconcile(); - - expect(pass.moved).toBe(1); + expect(await copyOne(doc)).toBe('moved'); const rows = await pgRowsFor(doc._id); expect(rows).toHaveLength(1); expect(rows[0].content).toBe(`${MARKER} a`); @@ -103,35 +103,24 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { expect(after.reconciledPgId).toBe(Number(rows[0].id)); }); - test('old unmarked history is never imported but an explicitly marked fallback moves', async () => { - const old = await insertFallback('old-history', { pgFallback: undefined, createdAt: new Date('2020-01-01') }); - const fallback = await insertFallback('marked-fallback'); - expect((await reconcile()).moved).toBe(1); - expect(await pgRowsFor(old._id)).toHaveLength(0); - expect(await pgRowsFor(fallback._id)).toHaveLength(1); - const unchanged = await MongoMessage.findById(old._id).lean(); - expect(unchanged.reconciledAt).toBeUndefined(); - expect(unchanged.reconcileFailedAt).toBeUndefined(); - }); - - test('each message appears once in the Postgres read after a reconcile, and a second pass moves nothing', async () => { + test('each message appears once in the Postgres read after a copy, and a second copy moves nothing', async () => { await insertFallback('b1'); await insertFallback('b2'); - await reconcile(); - const second = await reconcile(); + const docs = await MongoMessage.find({ content: { $regex: `^${MARKER} b` } }); + expect(docs).toHaveLength(2); + for (const doc of docs) await copyOne(doc); + for (const doc of docs) expect(await copyOne(doc)).toBe('present'); - expect(second.moved).toBe(0); const listed = await PGMessage.findByPodId(POD, 50); const ours = listed.filter((m) => String(m.content).startsWith(`${MARKER} b`)); expect(ours).toHaveLength(2); expect(new Set(ours.map((m) => m._id)).size).toBe(2); }); - test('two concurrent passes leave exactly one Postgres row per message', async () => { + test('two concurrent copies leave exactly one Postgres row per message', async () => { const doc = await insertFallback('c'); - await ensureLegacyIdIndex(); const [x, y] = await Promise.all([copyOne(doc), copyOne(doc)]); expect([x, y].filter((outcome) => outcome === 'moved')).toHaveLength(1); @@ -140,80 +129,6 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { expect(stamped.reconciledAt).toBeInstanceOf(Date); }); - test('another replica holding the index maintenance lock defers the pass', async () => { - const doc = await insertFallback('index-lock'); - const client = await pool.connect(); - try { - await client.query('SELECT pg_advisory_lock(313, 2)'); - expect((await reconcile()).moved).toBe(0); - expect(await pgRowsFor(doc._id)).toHaveLength(0); - } finally { - await client.query('SELECT pg_advisory_unlock(313, 2)'); - client.release(); - } - expect((await reconcile()).moved).toBe(1); - }); - - test('a pass is bounded by maxPerPass and the remainder moves on the next pass', async () => { - await insertFallback('d1'); - await insertFallback('d2'); - await insertFallback('d3'); - - const first = await reconcile({ maxPerPass: 2 }); - const second = await reconcile({ maxPerPass: 2 }); - - expect(first.moved).toBe(2); - expect(second.moved).toBe(1); - }); - - test('a missing PG pod is deferred without blocking later rows, then retries after sync', async () => { - const orphan = await insertFallback('e-orphan', { podId: new mongoose.Types.ObjectId().toHexString() }); - const doc = await insertFallback('e-ok'); - - const pass = await reconcile({ maxPerPass: 1 }); - const next = await reconcile({ maxPerPass: 1 }); - - expect(pass.skipped).toBe(1); - const refused = await MongoMessage.findById(orphan._id).lean(); - expect(refused.reconcileFailedAt).toBeUndefined(); - expect(refused.reconcileRetryAt).toBeInstanceOf(Date); - await pool.query(`INSERT INTO pods (id, name, type, created_by) VALUES ($1, 'late-pod', 'chat', $2)`, [String(orphan.podId), USER]); - await MongoMessage.updateOne({ _id: orphan._id }, { $set: { reconcileRetryAt: new Date(0) } }); - expect((await reconcile()).moved).toBe(1); - expect(await pgRowsFor(orphan._id)).toHaveLength(1); - await pool.query('DELETE FROM messages WHERE pod_id = $1', [String(orphan.podId)]); - await pool.query('DELETE FROM pods WHERE id = $1', [String(orphan.podId)]); - expect(await pgRowsFor(doc._id)).toHaveLength(1); - expect(next.moved).toBe(1); - }); - - test('more than a full batch of poison rows cannot starve a valid message', async () => { - const bad = Array.from({ length: 201 }, () => ({ - pgFallback: true, podId: new mongoose.Types.ObjectId(POD), userId: new mongoose.Types.ObjectId(USER), - content: `${MARKER} bad\0`, createdAt: new Date(0), - })); - await MongoMessage.collection.insertMany([...bad, - { pgFallback: true, content: `${MARKER} missing-pod`, userId: new mongoose.Types.ObjectId(USER), createdAt: new Date(0) }, - { pgFallback: true, content: `${MARKER} missing-user`, podId: new mongoose.Types.ObjectId(POD), createdAt: new Date(0) }, - ]); - const good = await insertFallback('after-poison'); - expect((await reconcile()).skipped).toBe(200); - expect((await reconcile()).moved).toBe(1); - expect(await pgRowsFor(good._id)).toHaveLength(1); - expect(await MongoMessage.countDocuments({ reconcileFailedAt: { $exists: true } })).toBe(203); - expect((await reconcile()).skipped).toBe(0); - }); - - test('retry-ready orphan rows do not crowd out unattempted messages', async () => { - await MongoMessage.collection.insertMany(Array.from({ length: 201 }, () => ({ - pgFallback: true, podId: new mongoose.Types.ObjectId(), userId: new mongoose.Types.ObjectId(USER), - content: `${MARKER} retry-orphan`, createdAt: new Date(0), reconcileRetryAt: new Date(0), - }))); - const good = await insertFallback('new-after-retries'); - expect((await reconcile({ maxPerPass: 1 })).moved).toBe(1); - expect(await pgRowsFor(good._id)).toHaveLength(1); - }); - test('a second copy leaves the first reconciledAt and pgId unchanged', async () => { const doc = await insertFallback('set-once'); await copyOne(doc); @@ -237,16 +152,16 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { expect(after.reconciledPgId).toBeUndefined(); }); - test('card payload and kernel authorship survive reconciliation', async () => { + test('card payload and kernel authorship survive the copy', async () => { const payload = { type: 'decision', options: ['continue'] }; const doc = await insertFallback('card', { messageType: 'card', payload, authorKind: 'kernel' }); - await reconcile(); + await copyOne(doc); expect((await pgRowsFor(doc._id))[0]).toMatchObject({ payload, author_kind: 'kernel' }); }); - test('reconciled IDs work as replies and thread roots and reject missing or cross-pod targets', async () => { + test('copied IDs work as replies and thread roots and reject missing or cross-pod targets', async () => { const doc = await insertFallback('thread-root'); - await reconcile(); + await copyOne(doc); const { resolveThreadRoot, assertReplyTargetInPod } = require('../../services/threadRootResolver'); const root = await resolveThreadRoot({ podId: POD, threadRootId: String(doc._id) }); await expect(assertReplyTargetInPod({ podId: POD, replyToMessageId: String(doc._id) })).resolves.toBeUndefined(); @@ -259,32 +174,13 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { test('all-digit ObjectIds resolve as Mongo IDs rather than PG integers', async () => { const doc = await insertFallback('digit-id', { _id: new mongoose.Types.ObjectId('123456789012345678901234') }); - await reconcile(); + await copyOne(doc); expect(await PGMessage.resolveId(String(doc._id))).toBe(String((await pgRowsFor(doc._id))[0].id)); }); - test('a pod that exists only in Mongo is backfilled and its message then moves', async () => { - const orphanPod = new mongoose.Types.ObjectId().toHexString(); - await mongoose.connection.db.collection('pods').insertOne({ - _id: new mongoose.Types.ObjectId(orphanPod), - name: 'reconcile-313-backfill', - type: 'chat', - createdBy: new mongoose.Types.ObjectId(USER), - members: [new mongoose.Types.ObjectId(USER)], - }); - const doc = await insertFallback('i', { podId: orphanPod }); - - const pass = await reconcile(); - - expect(pass.moved).toBe(1); - expect(await pgRowsFor(doc._id)).toHaveLength(1); - expect((await MongoMessage.findById(doc._id).lean()).reconcileFailedAt).toBeUndefined(); - await pool.query('DELETE FROM pods WHERE id = $1', [orphanPod]); - }); - test('a moved message resolves its Mongo id for delete', async () => { const doc = await insertFallback('f'); - await reconcile(); + await copyOne(doc); const pgId = await PGMessage.resolveId(String(doc._id)); expect(pgId).toBe(String((await pgRowsFor(doc._id))[0].id)); @@ -293,7 +189,7 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { expect(await pgRowsFor(doc._id)).toHaveLength(0); }); - test.each(['mongo', 'pg'])('delete by %s ID removes the unstamped availability copy before another pass', async (idKind) => { + test.each(['mongo', 'pg'])('delete by %s ID removes the unstamped Mongo copy too', async (idKind) => { const doc = await insertFallback('crashed-stamp-delete'); await copyOne(doc); const pgId = await PGMessage.resolveId(String(doc._id)); @@ -304,13 +200,12 @@ d('TASK-313 part 2: reconcile Mongo fallback messages into PostgreSQL', () => { expect(res.json).toHaveBeenCalledWith({ msg: 'Message deleted' }); expect(await MongoMessage.findById(doc._id).lean()).toBeNull(); expect(await pgRowsFor(doc._id)).toHaveLength(0); - expect((await reconcile()).moved).toBe(0); expect(await pgRowsFor(doc._id)).toHaveLength(0); }); test('a moved message resolves its Mongo id for reactions', async () => { const doc = await insertFallback('g'); - await reconcile(); + await copyOne(doc); const pgId = await PGMessage.resolveId(String(doc._id)); await MessageReaction.add(pgId, USER, '๐Ÿ‘'); diff --git a/backend/__tests__/service/messages.test.js b/backend/__tests__/service/messages.test.js index 9e27d58dea..a6d0ce43ef 100644 --- a/backend/__tests__/service/messages.test.js +++ b/backend/__tests__/service/messages.test.js @@ -23,8 +23,11 @@ jest.mock('../../models/pg/Message'); describe('Message Routes Integration Tests', () => { let app; + const priorPgHost = process.env.PG_HOST; beforeAll(async () => { + // PGMessage is mocked; PG_HOST selects the PG read (TASK-503). + process.env.PG_HOST = process.env.PG_HOST || 'pg.mocked'; await setupMongoDb(); app = express(); app.use(express.json()); @@ -33,6 +36,8 @@ describe('Message Routes Integration Tests', () => { }); afterAll(async () => { + if (priorPgHost === undefined) delete process.env.PG_HOST; + else process.env.PG_HOST = priorPgHost; await closeMongoDb(); }); diff --git a/backend/__tests__/unit/controllers/messageController.test.js b/backend/__tests__/unit/controllers/messageController.test.js index 08eb544156..8250a99523 100644 --- a/backend/__tests__/unit/controllers/messageController.test.js +++ b/backend/__tests__/unit/controllers/messageController.test.js @@ -63,6 +63,14 @@ describe('messageController', () => { }); describe('getMessages', () => { + // Reads follow PG_HOST like the writes; the unit setup unsets it. + const OLD_PG_HOST = process.env.PG_HOST; + beforeEach(() => { process.env.PG_HOST = 'localhost'; }); + afterEach(() => { + if (OLD_PG_HOST === undefined) delete process.env.PG_HOST; + else process.env.PG_HOST = OLD_PG_HOST; + }); + it('returns 400 if podId is missing', async () => { const req = { params: {}, query: {}, user: { id: 'u1' } }; const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; @@ -136,8 +144,8 @@ describe('messageController', () => { expect(gateWelcomeHostedActions).toHaveBeenCalledWith(page, 'p1', 'u1'); expect(res.json).toHaveBeenCalledWith(gated); - // The Mongo fallback serves through the same gate. - PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + // The Mongo store (no PG_HOST) serves through the same gate. + delete process.env.PG_HOST; MongoMessage.find.mockReturnValue({ populate: () => ({ sort: () => ({ limit: () => ({ lean: async () => [] }) }) }), }); @@ -668,6 +676,13 @@ describe('messageController', () => { }); describe('deleteMessage', () => { + const OLD_PG_HOST = process.env.PG_HOST; + beforeEach(() => { process.env.PG_HOST = 'localhost'; }); + afterEach(() => { + if (OLD_PG_HOST === undefined) delete process.env.PG_HOST; + else process.env.PG_HOST = OLD_PG_HOST; + }); + it('returns 404 when message not found', async () => { PGMessage.resolveId.mockResolvedValue(null); PGMessage.findById.mockResolvedValue(null); @@ -677,7 +692,7 @@ describe('messageController', () => { expect(res.status).toHaveBeenCalledWith(404); }); - it('deletes the resolved PG row and availability copy for a reconciled Mongo id', async () => { + it('deletes the resolved PG row and the legacy copy for a copied Mongo id', async () => { PGMessage.resolveId.mockResolvedValue('41'); PGMessage.findById.mockResolvedValue({ user_id: 'u1', pod_id: 'p1' }); const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; @@ -690,23 +705,36 @@ describe('messageController', () => { expect(res.json).toHaveBeenCalledWith({ msg: 'Message deleted' }); }); - it('refuses deletion of a reconciled row while PG is down without deleting either copy', async () => { + it('with PG down, answers a retryable 503 and touches neither store', async () => { PGMessage.resolveId.mockRejectedValue(new Error('PG unavailable')); - MongoMessage.findById.mockResolvedValue({ reconciledAt: new Date(), userId: 'u1', podId: 'p1' }); + MongoMessage.findById.mockResolvedValue({ _id: 'm1', userId: 'u1', podId: 'p1' }); const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; await messageController.deleteMessage({ params: { id: '123456789012345678901234' }, userId: 'u1' }, res); expect(res.status).toHaveBeenCalledWith(503); + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'MESSAGE_STORE_UNAVAILABLE' })); + expect(MongoMessage.findById).not.toHaveBeenCalled(); expect(PGMessage.delete).not.toHaveBeenCalled(); expect(MongoMessage.deleteOne).not.toHaveBeenCalled(); expect(MongoMessage.findByIdAndDelete).not.toHaveBeenCalled(); }); - it('deletes an unreconciled Mongo-only row', async () => { + it('with PG configured, a Mongo-only row PG never had is not found', async () => { PGMessage.resolveId.mockResolvedValue(null); MongoMessage.findById.mockResolvedValue({ _id: 'mongo-only', userId: 'u1', podId: 'p1' }); const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; await messageController.deleteMessage({ params: { id: 'mongo-only' }, userId: 'u1' }, res); + expect(res.status).toHaveBeenCalledWith(404); + expect(MongoMessage.findById).not.toHaveBeenCalled(); + expect(MongoMessage.findByIdAndDelete).not.toHaveBeenCalled(); + }); + + it('without PG configured, deletes the Mongo row and never asks PG', async () => { + delete process.env.PG_HOST; + MongoMessage.findById.mockResolvedValue({ _id: 'mongo-only', userId: 'u1', podId: 'p1' }); + const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; + await messageController.deleteMessage({ params: { id: 'mongo-only' }, userId: 'u1' }, res); expect(MongoMessage.findByIdAndDelete).toHaveBeenCalledWith('mongo-only'); + expect(PGMessage.resolveId).not.toHaveBeenCalled(); expect(PGMessage.delete).not.toHaveBeenCalled(); }); diff --git a/backend/__tests__/unit/repositories/messageRepository.test.js b/backend/__tests__/unit/repositories/messageRepository.test.js index 0c77c9a9a8..a46ec55216 100644 --- a/backend/__tests__/unit/repositories/messageRepository.test.js +++ b/backend/__tests__/unit/repositories/messageRepository.test.js @@ -1,6 +1,6 @@ /** - * TASK-503: the one pod-message read. PG answers when it can; the MongoDB - * fallback lives only here and returns the PG row shape, oldest first. + * TASK-503: the one pod-message read. With PG_HOST set it reads PG only; without + * it, MongoDB only, in the PG row shape, oldest first. */ jest.mock('../../../models/pg/Message', () => ({ findByPodId: jest.fn() })); jest.mock('../../../models/Message', () => ({ find: jest.fn() })); @@ -22,11 +22,15 @@ const mongoChain = (docs) => { }; describe('listPodMessages', () => { + const priorPgHost = process.env.PG_HOST; beforeEach(() => { jest.clearAllMocks(); - jest.spyOn(console, 'warn').mockImplementation(() => {}); + process.env.PG_HOST = 'pg.test'; + }); + afterEach(() => { + if (priorPgHost === undefined) delete process.env.PG_HOST; + else process.env.PG_HOST = priorPgHost; }); - afterEach(() => console.warn.mockRestore()); it('returns the PG rows as they are and never reads MongoDB', async () => { const rows = [{ id: '1' }, { id: '2' }]; @@ -36,25 +40,33 @@ describe('listPodMessages', () => { expect(MongoMessage.find).not.toHaveBeenCalled(); }); + it('with PG configured, a PG failure is an error and MongoDB is never read', async () => { + PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + mongoChain([{ _id: 'm1', podId: 'p1', userId: 'u1', content: 'stale', createdAt: new Date() }]); + await expect(listPodMessages('p1')).rejects.toThrow('pg down'); + expect(MongoMessage.find).not.toHaveBeenCalled(); + }); + it('clamps the limit to 1..200 for both stores', async () => { PGMessage.findByPodId.mockResolvedValue([]); await listPodMessages('p1', { limit: 5000 }); expect(PGMessage.findByPodId).toHaveBeenLastCalledWith('p1', 200, null); - PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + delete process.env.PG_HOST; const calls = mongoChain([]); await listPodMessages('p1', { limit: 5000 }); expect(calls.limit).toBe(200); }); - it('on a PG failure reads MongoDB, newest page first, and returns it oldest first', async () => { - PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + it('without PG configured, reads MongoDB only, newest page first, and returns it oldest first', async () => { + delete process.env.PG_HOST; const calls = mongoChain([ { _id: 'm2', podId: 'p1', userId: 'u2', content: 'second', createdAt: new Date('2026-01-02') }, { _id: 'm1', podId: 'p1', userId: 'u1', content: 'first', createdAt: new Date('2026-01-01') }, ]); const rows = await listPodMessages('p1', { limit: 2, before: '2026-02-01T00:00:00.000Z' }); expect(rows.map((r) => r.id)).toEqual(['m1', 'm2']); + expect(PGMessage.findByPodId).not.toHaveBeenCalled(); expect(calls.query).toEqual({ podId: 'p1', createdAt: { $lt: new Date('2026-02-01T00:00:00.000Z') } }); expect(calls.sort).toEqual({ createdAt: -1 }); expect(calls.populate).toEqual(['userId', 'username profilePicture isBot']); diff --git a/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js index 63f4f1dfff..d5a7a4f208 100644 --- a/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js +++ b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js @@ -1,4 +1,4 @@ -jest.mock('../../../services/pgMessageReconcileService', () => ({ copyOne: jest.fn(), ensureLegacyIdIndex: jest.fn() })); +jest.mock('../../../services/legacyMessageCopy', () => ({ copyOne: jest.fn(), ensureLegacyIdIndex: jest.fn() })); const { backfillLocalMessages, main, parseArgs } = require('../../../scripts/backfill-local-messages-to-pg'); diff --git a/backend/__tests__/unit/server.test.js b/backend/__tests__/unit/server.test.js index ce461aebc3..495bd22be7 100644 --- a/backend/__tests__/unit/server.test.js +++ b/backend/__tests__/unit/server.test.js @@ -59,11 +59,6 @@ jest.mock('../../services/agentInstallationCleanupService', () => { const initInstallationCleanup = mockInitInstallationCleanup; return { initInstallationCleanup }; }); -let mockInitPgMessageReconcile = jest.fn(); -jest.mock('../../services/pgMessageReconcileService', () => { - const initPgMessageReconcile = mockInitPgMessageReconcile; - return { initPgMessageReconcile }; -}); jest.mock('../../services/schedulerService', () => ({ start: jest.fn() })); jest.mock('../../services/discordGatewayService', () => ({ start: jest.fn() })); @@ -109,7 +104,6 @@ describe('server pg boot routes', () => { mockInitPGDB = jest.fn(); mockInitPgRetention = jest.fn(); mockInitInstallationCleanup = jest.fn(); - mockInitPgMessageReconcile = jest.fn(); }); afterEach(() => { @@ -203,7 +197,6 @@ describe('server pg boot routes', () => { await request(app).get('/api/pg/messages').expect(200); expect(mockInitPgRetention).toHaveBeenCalledTimes(shouldStart ? 1 : 0); expect(mockInitInstallationCleanup).toHaveBeenCalledTimes(shouldStart ? 1 : 0); - expect(mockInitPgMessageReconcile).toHaveBeenCalledTimes(shouldStart ? 1 : 0); } finally { if (priorNodeEnv === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = priorNodeEnv; diff --git a/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js b/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js index 8b6e866bbf..21bfbad6ee 100644 --- a/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js +++ b/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js @@ -1,8 +1,7 @@ /** * Who posted in a pod since a time: the assigned-task wake reads this to decide - * whether a person acted. On PostgreSQL a post that fell back to MongoDB is only - * in MongoDB until it is reconciled, so both stores are read. One store failing - * keeps the other's authors; both failing throws, and the wake fails closed. + * whether a person acted. One store answers, by the PG_HOST gate the message + * writes use (TASK-503). A failed read throws, and the wake fails closed. */ const AgentMessageService = require('../../../services/agentMessageService'); const Message = require('../../../models/Message'); @@ -24,36 +23,22 @@ describe('authorIdsSince', () => { else process.env.PG_HOST = origPgHost; }); - test('on PostgreSQL, includes an author whose post is still in the MongoDB fallback', async () => { + test('on PostgreSQL, reads PostgreSQL only', async () => { process.env.PG_HOST = 'localhost'; - PGMessage.findAuthorIdsSince.mockResolvedValue(['agent-1', 'person-1']); - Message.distinct.mockResolvedValue(['person-2', 'person-1']); - const ids = await AgentMessageService.authorIdsSince('pod-1', SINCE); - expect(ids.sort()).toEqual(['agent-1', 'person-1', 'person-2']); - expect(Message.distinct).toHaveBeenCalledWith('userId', expect.objectContaining({ createdAt: { $gt: SINCE } })); - }); - - // An author either store returns did post. During a PostgreSQL outage the - // posts land in MongoDB, so that read is the one that sees them. - test('a failed PostgreSQL read keeps the MongoDB authors', async () => { - process.env.PG_HOST = 'localhost'; - PGMessage.findAuthorIdsSince.mockRejectedValue(new Error('pg down')); + PGMessage.findAuthorIdsSince.mockResolvedValue(['agent-1', 'person-1', 'person-1']); Message.distinct.mockResolvedValue(['person-2']); - expect(await AgentMessageService.authorIdsSince('pod-1', SINCE)).toEqual(['person-2']); + expect(await AgentMessageService.authorIdsSince('pod-1', SINCE)).toEqual(['agent-1', 'person-1']); + expect(Message.distinct).not.toHaveBeenCalled(); }); - test('a failed MongoDB read keeps the PostgreSQL authors', async () => { - process.env.PG_HOST = 'localhost'; - PGMessage.findAuthorIdsSince.mockResolvedValue(['person-1']); - Message.distinct.mockRejectedValue(new Error('mongo down')); - expect(await AgentMessageService.authorIdsSince('pod-1', SINCE)).toEqual(['person-1']); - }); - - test('both reads failing throws', async () => { + // With PostgreSQL configured nothing writes MongoDB messages (TASK-503), so + // MongoDB cannot stand in for a failed read. The wake fails closed instead. + test('a failed PostgreSQL read throws and does not read MongoDB', async () => { process.env.PG_HOST = 'localhost'; PGMessage.findAuthorIdsSince.mockRejectedValue(new Error('pg down')); - Message.distinct.mockRejectedValue(new Error('mongo down')); + Message.distinct.mockResolvedValue(['person-2']); await expect(AgentMessageService.authorIdsSince('pod-1', SINCE)).rejects.toThrow('pg down'); + expect(Message.distinct).not.toHaveBeenCalled(); }); test('without PostgreSQL, reads MongoDB only', async () => { diff --git a/backend/__tests__/unit/services/agentMessageService.threadReads.test.js b/backend/__tests__/unit/services/agentMessageService.threadReads.test.js index b6b82c2aa9..a097d4a7a0 100644 --- a/backend/__tests__/unit/services/agentMessageService.threadReads.test.js +++ b/backend/__tests__/unit/services/agentMessageService.threadReads.test.js @@ -11,10 +11,11 @@ * * - every PG message carries the two threading columns (explicit null for * "not in a thread" โ€” an absent key means "server cannot say", which is - * the Mongo-fallback shape, same convention as the frontend threadView); + * the Mongo-store shape, same convention as the frontend threadView); * - `threadRootId` scopes the read and reaches the model verbatim; - * - the Mongo fallback REFUSES a thread-scoped read rather than returning - * the whole pod as if it were the thread. + * - with PG configured, a PG failure surfaces; there is no Mongo fallback; + * - the Mongo store (no PG_HOST) REFUSES a thread-scoped read rather than + * returning the whole pod as if it were the thread. */ const AgentMessageService = require('../../../services/agentMessageService'); @@ -83,11 +84,21 @@ describe('getRecentMessages โ€” thread structure on the PG path', () => { expect(PGMessage.findByPodId).toHaveBeenCalledWith(POD, 20, null, '57577'); }); - test('the Mongo fallback refuses a thread-scoped read instead of faking it', async () => { + test('with PG configured, a PG failure surfaces instead of reading MongoDB', async () => { PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + await expect( + AgentMessageService.getRecentMessages(POD, 20, undefined, undefined, '57577'), + ).rejects.toThrow('pg down'); + await expect(AgentMessageService.getRecentMessages(POD, 20)).rejects.toThrow('pg down'); + }); + + test('the Mongo store refuses a thread-scoped read instead of faking it', async () => { + delete process.env.PG_HOST; + await expect( AgentMessageService.getRecentMessages(POD, 20, undefined, undefined, '57577'), ).rejects.toThrow('threadRootId reads require the PostgreSQL message store'); + expect(PGMessage.findByPodId).not.toHaveBeenCalled(); }); }); diff --git a/backend/__tests__/unit/services/nativeRuntimeService.hostedBroker.test.js b/backend/__tests__/unit/services/nativeRuntimeService.hostedBroker.test.js index aa86af0466..bb28278efa 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.hostedBroker.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.hostedBroker.test.js @@ -80,6 +80,9 @@ jest.mock('../../../services/agentTypingService', () => ({ jest.mock('../../../services/agentMessageService', () => ({ postMessage: jest.fn().mockResolvedValue({ success: true }), })); +// The pod-message read follows PG_HOST (TASK-503) and the unit setup unsets +// it, so name a host for the mocked PG model below to answer. +process.env.PG_HOST = 'pg.test'; jest.mock('../../../models/pg/Message', () => ({ findByPodId: jest.fn().mockResolvedValue([]), })); diff --git a/backend/__tests__/unit/services/nativeRuntimeService.recentMessages.test.js b/backend/__tests__/unit/services/nativeRuntimeService.recentMessages.test.js index 01ac9c4ba0..f32c35d555 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.recentMessages.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.recentMessages.test.js @@ -84,10 +84,14 @@ const sentUserMessage = () => { }; describe('native runtime: recent pod messages inline (recentMessages)', () => { - const previousEnv = { baseUrl: process.env.LITELLM_BASE_URL, masterKey: process.env.LITELLM_MASTER_KEY }; + const previousEnv = { + baseUrl: process.env.LITELLM_BASE_URL, masterKey: process.env.LITELLM_MASTER_KEY, pgHost: process.env.PG_HOST, + }; beforeEach(() => { jest.clearAllMocks(); + // The pod-message read follows PG_HOST; the unit setup unsets it. + process.env.PG_HOST = 'pg.test'; process.env.LITELLM_BASE_URL = 'http://litellm.test'; process.env.LITELLM_MASTER_KEY = 'test-key'; AgentRun.create.mockImplementation(async (doc) => ({ @@ -108,6 +112,8 @@ describe('native runtime: recent pod messages inline (recentMessages)', () => { else process.env.LITELLM_BASE_URL = previousEnv.baseUrl; if (previousEnv.masterKey === undefined) delete process.env.LITELLM_MASTER_KEY; else process.env.LITELLM_MASTER_KEY = previousEnv.masterKey; + if (previousEnv.pgHost === undefined) delete process.env.PG_HOST; + else process.env.PG_HOST = previousEnv.pgHost; }); test('an opted-in turn opens with the prior messages, oldest first, and leaves out the one it answers', async () => { diff --git a/backend/__tests__/unit/services/nativeRuntimeService.webFetch.test.js b/backend/__tests__/unit/services/nativeRuntimeService.webFetch.test.js index 25930d0fbe..d6ef58405e 100644 --- a/backend/__tests__/unit/services/nativeRuntimeService.webFetch.test.js +++ b/backend/__tests__/unit/services/nativeRuntimeService.webFetch.test.js @@ -75,6 +75,9 @@ jest.mock('../../../services/outboundUrlGuard', () => ({ getFromPublicAddress: (...args) => mockGetPage(...args), })); const mockRecentRows = jest.fn(); +// The pod-message read follows PG_HOST (TASK-503) and the unit setup unsets +// it, so name a host for the mocked PG model below to answer. +process.env.PG_HOST = 'pg.test'; jest.mock('../../../models/pg/Message', () => ({ findByPodId: (...args) => mockRecentRows(...args) })); // The Mongo fallback, read when the PG mirror is down. const mockMongoDocs = jest.fn(); @@ -427,8 +430,8 @@ describe('native turn with commonly_fetch_url', () => { expect(toolMessages(1)[1].content).toMatch(/does not appear in this conversation/); }); - test('on the Mongo fallback, a human\'s post gives a link and an agent\'s does not', async () => { - mockRecentRows.mockRejectedValue(new Error('pg down')); + test('on the Mongo store (no PG_HOST), a human\'s post gives a link and an agent\'s does not', async () => { + delete process.env.PG_HOST; mockMongoDocs.mockResolvedValue([ { _id: 'm6', userId: { _id: '507f1f77bcf86cd7994390bb', username: 'scout', isBot: true }, content: 'https://attacker.example/y', createdAt: new Date() }, { _id: 'm5', userId: { _id: SAM_ID, username: 'sam', isBot: false }, content: 'https://github.com/acme/other', createdAt: new Date() }, @@ -444,6 +447,8 @@ describe('native turn with commonly_fetch_url', () => { await runAgent(install(), mention('@analyst summarize')); expect(mockGetPage.mock.calls).toEqual([['https://github.com/acme/other']]); + expect(mockRecentRows).not.toHaveBeenCalled(); + process.env.PG_HOST = 'pg.test'; }); // The message a turn answers gave diff --git a/backend/controllers/messageController.ts b/backend/controllers/messageController.ts index d80b7bdcd8..bfd4690362 100644 --- a/backend/controllers/messageController.ts +++ b/backend/controllers/messageController.ts @@ -450,30 +450,27 @@ exports.createMessage = async (req: AuthRequest, res: Response): Promise = exports.deleteMessage = async (req: AuthRequest, res: Response): Promise => { try { - // A message the reconciler moved answers to its Mongo id, and the PG row - // is the one to delete. Deleting only the Mongo copy would leave the - // message visible in PG. + // With PG_HOST set, PostgreSQL is the only message store (TASK-503). A + // copied legacy message answers to its Mongo id through resolveId, and the + // PG row is the one to delete. PG unreachable is a retryable 503, never a + // delete of the Mongo copy that would leave the message visible in PG. const messageId = req.params.id; let message: NormalizedMessage | null = null; let pgId: string | null = null; - let pgUnavailable = false; - try { - pgId = await PGMessage.resolveId(messageId); - if (pgId) message = await PGMessage.findById(pgId) as NormalizedMessage | null; - } catch { - pgUnavailable = true; - } - if (!pgId) { - const mongoMsg = await MongoMessage.findById(messageId) as Record | null; - if (mongoMsg?.reconciledAt && pgUnavailable) { - // The PG row exists and cannot be reached. Deleting the Mongo copy - // now would leave it visible once PG returns. + if (process.env.PG_HOST) { + try { + pgId = await PGMessage.resolveId(messageId); + if (pgId) message = await PGMessage.findById(pgId) as NormalizedMessage | null; + } catch (pgError) { + console.warn('[messageController] PG delete lookup failed:', (pgError as Error).message); res.status(503).json({ - error: 'Message is temporarily unavailable. Please try again shortly.', - code: 'MESSAGE_REQUIRES_POSTGRES', + error: 'Messages are temporarily unavailable. Please try again shortly.', + code: 'MESSAGE_STORE_UNAVAILABLE', }); return; } + } else { + const mongoMsg = await MongoMessage.findById(messageId) as Record | null; if (mongoMsg) message = normalizeMongo(mongoMsg); } @@ -498,8 +495,8 @@ exports.deleteMessage = async (req: AuthRequest, res: Response): Promise = if (pgId) { await PGMessage.delete(pgId); - // Keep the availability copy from resurrecting a deleted message during - // the next PG outage. Cleanup cannot turn a committed delete into a 500. + // Remove the legacy Mongo copy too, so the Mongo-only readers (stats, + // activity) stop counting it. Cleanup cannot turn a committed delete into a 500. try { const copyKeys: Array> = [{ reconciledPgId: Number(pgId) }]; // PG can commit before its Mongo stamp. The stable legacy ID still @@ -510,7 +507,7 @@ exports.deleteMessage = async (req: AuthRequest, res: Response): Promise = } await MongoMessage.deleteOne({ $or: copyKeys }); } catch (cleanupError) { - console.warn('[messageController] fallback delete cleanup failed:', (cleanupError as Error).message); + console.warn('[messageController] legacy copy delete cleanup failed:', (cleanupError as Error).message); } } else { await MongoMessage.findByIdAndDelete(messageId); diff --git a/backend/models/Message.ts b/backend/models/Message.ts index 2c66dda65d..f38d7c768f 100644 --- a/backend/models/Message.ts +++ b/backend/models/Message.ts @@ -25,17 +25,16 @@ export interface IMessage extends Document { updatedAt: Date; freshnessPending?: boolean; freshnessCommittedAt?: Date; - // TASK-313 part 2. Set once the reconciler has written this message into - // Postgres (reconciledPgId is the PG row). Set-once: the reconciler's - // updateOne filters on its absence, so concurrent passes cannot re-stamp. + // Set once `copyOne` (services/legacyMessageCopy) has written this message + // into Postgres (reconciledPgId is the PG row). Set-once: its updateOne + // filters on its absence, so concurrent copies cannot re-stamp. reconciledAt?: Date; reconciledPgId?: number; - // A permanent malformed-data refusal. Excluded from the - // reconciler's scan so one such row cannot hold back the rows behind it. + // Written by the removed TASK-313 reconciler; kept so existing rows load. reconcileFailedAt?: Date; reconcileError?: string; reconcileRetryAt?: Date; - /** Written only by the availability path, never inferred for legacy history. */ + /** Written by the removed PG-outage fallback (TASK-503); history only. */ pgFallback?: boolean; } diff --git a/backend/repositories/messageRepository.ts b/backend/repositories/messageRepository.ts index 2cac872e32..a1439325c0 100644 --- a/backend/repositories/messageRepository.ts +++ b/backend/repositories/messageRepository.ts @@ -1,11 +1,12 @@ /** - * Pod message reads (TASK-503). PostgreSQL is the message store; MongoDB holds - * only legacy rows and the `pgFallback` rows written while PG was down. + * Pod message reads (TASK-503). With PG_HOST set, PostgreSQL is the only + * message store: every write path goes there, and a PG read failure surfaces + * as an error rather than a silent read from MongoDB, whose rows the PG read + * never shows. Without PG_HOST, MongoDB is the instance's only store, the same + * gate the write paths use. * - * Every read that pages a pod's messages goes through `listPodMessages`, so the - * Mongo fallback lives here and nowhere else. Removing it once the 30-day - * trigger check clears is a change to this file only; a PG failure then - * surfaces as an error instead of a silent read from the other store. + * Every read that pages a pod's messages goes through `listPodMessages`, so + * that choice lives here and nowhere else. * * Callers get one row shape whichever store answered: the shape of * `models/pg/Message` `findByPodId`, oldest first. @@ -51,7 +52,7 @@ export interface ListPodMessagesOptions { before?: string | null; } -/** Same clamp as the PG read, so the fallback can never page more rows. */ +/** Same clamp as the PG read, so the Mongo read can never page more rows. */ const clampLimit = (limit: unknown): number => Math.min(200, Math.max(1, Number.parseInt(String(limit), 10) || 50)); type MongoAuthor = { _id?: { toString(): string }; username?: string; profilePicture?: string; isBot?: boolean }; @@ -104,10 +105,8 @@ export async function listPodMessages( { limit = 50, before = null }: ListPodMessagesOptions = {}, ): Promise { const safeLimit = clampLimit(limit); - try { + if (process.env.PG_HOST) { return await PGMessage.findByPodId(podId, safeLimit, before || null) as PodMessage[]; - } catch (pgError) { - console.warn('[messages] PG read failed, falling back to MongoDB:', (pgError as Error).message); } const query: Record = { podId }; if (before) query.createdAt = { $lt: new Date(before) }; diff --git a/backend/scripts/backfill-local-messages-to-pg.ts b/backend/scripts/backfill-local-messages-to-pg.ts index e3ea5ed555..d62a76701c 100644 --- a/backend/scripts/backfill-local-messages-to-pg.ts +++ b/backend/scripts/backfill-local-messages-to-pg.ts @@ -5,11 +5,11 @@ * The local profile (docker-compose.local.yml) used to run without Postgres, * so every chat message lived in Mongo. Once Postgres is healthy, chat reads * Postgres only and never merges the two stores, so that history disappears - * from the UI. The background reconciler (pgMessageReconcileService) does not - * help: it moves only rows flagged `pgFallback`, written while PG was down. + * from the UI. Nothing else moves it: the TASK-313 background reconciler is + * gone, and it only ever moved rows flagged `pgFallback`. * - * This copies every not-yet-reconciled Mongo message through the reconciler's - * own `copyOne`, so it inherits its guarantees: the PG insert is ON CONFLICT + * This copies every not-yet-reconciled Mongo message through `copyOne` + * (services/legacyMessageCopy), so it inherits its guarantees: the PG insert is ON CONFLICT * on `legacy_mongo_id` and the Mongo row is stamped `reconciledAt` afterwards, * so a rerun or a crash mid-run inserts nothing twice. * @@ -48,7 +48,7 @@ /* eslint-disable no-console, no-await-in-loop */ import mongoose from 'mongoose'; -import { copyOne, ensureLegacyIdIndex } from '../services/pgMessageReconcileService'; +import { copyOne, ensureLegacyIdIndex } from '../services/legacyMessageCopy'; type CopyResult = 'moved' | 'present' | 'orphaned'; diff --git a/backend/server.ts b/backend/server.ts index 7f0305b52e..d75a02a183 100644 --- a/backend/server.ts +++ b/backend/server.ts @@ -428,8 +428,6 @@ if (process.env.PG_HOST) { retentionErr?.message || retentionErr, ); } - // TASK-313 part 2: moves Mongo fallback messages into PG once it is back. - require('./services/pgMessageReconcileService').initPgMessageReconcile(); try { require('./services/agentInstallationCleanupService').initInstallationCleanup(); } catch (cleanupErr: any) { diff --git a/backend/services/agentMessageService.ts b/backend/services/agentMessageService.ts index f412b79767..c63169ceb0 100644 --- a/backend/services/agentMessageService.ts +++ b/backend/services/agentMessageService.ts @@ -2204,7 +2204,7 @@ class AgentMessageService { * is how a client detects an older server (see #757). * * `before` is an exclusive timestamp cursor applied identically on the PG - * and Mongo fallback paths. It must already be a validated epoch timestamp + * and Mongo paths. It must already be a validated epoch timestamp * so raw request values never cross this service boundary into a database * query and the query value cannot carry a user-controlled object shape. * Callers may request one extra row to determine whether an older page @@ -2212,25 +2212,18 @@ class AgentMessageService { * internal consumers. */ static async authorIdsSince(podId: unknown, since: Date): Promise { - const mongo = Message.distinct('userId', { - podId: { $eq: podId }, createdAt: { $gt: since }, messageType: { $ne: 'system' }, - }) as Promise; - if (!PGMessage || !process.env.PG_HOST) return (await mongo).map(String); - // A post that fell back to MongoDB while PostgreSQL was failing is only - // there until it is reconciled, so read both. An author either store - // returns did post, so one store failing keeps the other's authors; only - // both failing leaves nothing to answer with. - const [pg, fallback] = await Promise.allSettled([ - (PGMessage as { findAuthorIdsSince(id: unknown, at: Date): Promise }).findAuthorIdsSince(podId, since), - mongo, - ]); - if (pg.status === 'rejected' && fallback.status === 'rejected') throw pg.reason; - const ids: string[] = []; - for (const [store, read] of [['postgres', pg], ['mongo', fallback]] as const) { - if (read.status === 'fulfilled') ids.push(...read.value.map(String)); - else console.warn(`[agent-message] ${store} author read failed:`, String(podId), (read.reason as Error)?.message); + // One store answers, by the same PG_HOST gate as the writes: with PG + // configured nothing writes Mongo messages, so a PG failure throws and the + // caller fails closed rather than reading a store that has no new posts. + if (PGMessage && process.env.PG_HOST) { + const ids = await (PGMessage as { findAuthorIdsSince(id: unknown, at: Date): Promise }) + .findAuthorIdsSince(podId, since); + return [...new Set(ids.map(String))]; } - return [...new Set(ids)]; + const ids = await Message.distinct('userId', { + podId: { $eq: podId }, createdAt: { $gt: since }, messageType: { $ne: 'system' }, + }) as unknown[]; + return ids.map(String); } static async getRecentMessages( @@ -2238,9 +2231,9 @@ class AgentMessageService { limit = 20, selfUserId?: unknown, beforeMs?: number, - // PG-only thread scoping. On the Mongo fallback (no threading columns) - // this filter cannot be honored, so rather than silently returning the - // whole pod as if it were the thread, the fallback throws โ€” an agent + // PG-only thread scoping. On the Mongo store (no PG_HOST, no threading + // columns) this filter cannot be honored, so rather than silently returning + // the whole pod as if it were the thread, the Mongo read throws โ€” an agent // that asked for a thread must never mistake the pod for it. threadRootId?: string, ): Promise { @@ -2261,63 +2254,59 @@ class AgentMessageService { ); if (PGMessage && process.env.PG_HOST) { - try { - const messages: Array> = await (PGMessage as { - findByPodId( - id: string, - limit: number, - before?: string | null, - threadRootId?: string | null, - ): Promise>>; - }).findByPodId( - String(podId), - limit, - before?.toISOString() ?? null, - threadRootId ?? null, - ); + const messages: Array> = await (PGMessage as { + findByPodId( + id: string, + limit: number, + before?: string | null, + threadRootId?: string | null, + ): Promise>>; + }).findByPodId( + String(podId), + limit, + before?.toISOString() ?? null, + threadRootId ?? null, + ); - const normalized = messages.map((msg) => { - const username = (msg.username as string) || 'Unknown'; - // Trust the mirrored `is_bot` column only. This used to fall back to - // a username substring test ('-bot', endsWith 'bot', 'openclaw-'), - // which misclassified any human named e.g. "talbot" or "abbot" as a - // bot โ€” and `isBot` gates the wrapper's echo suppression, so a false - // positive silently ate that agent's reply. - const isBot = msg.is_bot === true || msg.is_bot === 'true'; - return { - _id: msg.id, - id: msg.id, - content: msg.content as string, - messageType: (msg.message_type as string) || 'text', - userId: { - _id: msg.user_id, - username, - profilePicture: msg.profile_picture as string | undefined, - }, + const normalized = messages.map((msg) => { + const username = (msg.username as string) || 'Unknown'; + // Trust the mirrored `is_bot` column only. This used to fall back to + // a username substring test ('-bot', endsWith 'bot', 'openclaw-'), + // which misclassified any human named e.g. "talbot" or "abbot" as a + // bot โ€” and `isBot` gates the wrapper's echo suppression, so a false + // positive silently ate that agent's reply. + const isBot = msg.is_bot === true || msg.is_bot === 'true'; + return { + _id: msg.id, + id: msg.id, + content: msg.content as string, + messageType: (msg.message_type as string) || 'text', + userId: { + _id: msg.user_id, username, - isBot, - ...withSelf(msg.user_id), - createdAt: msg.created_at as Date, - // Threading structure for agent readers (TASK-052's read half): - // the SELECT has carried these since #1106; this mapper was the - // last literal dropping them, which is why the cue teaching - // "continue in a thread" was held โ€” a peer agent reading context - // could not see that a thread existed. Explicit null = "not in a - // thread"; the Mongo fallback below omits the keys entirely = - // "this server cannot say" (same convention as the frontend). - reply_to_message_id: (msg.reply_to_message_id as string | number | null) ?? null, - thread_root_id: (msg.thread_root_id as string | number | null) ?? null, - author_kind: (msg.author_kind as 'kernel' | null) ?? null, - replyTo: (msg.replyTo as MessageNormalized['replyTo']) ?? null, - }; - }); - // Internal provenance for send-time read acknowledgement. A Mongo - // fallback may omit PG-only messages and must not clear a PG cursor. - Object.defineProperty(normalized, 'readSource', { value: 'postgres' }); - return normalized; - } catch (pgError) { - console.error('PostgreSQL message fetch failed, falling back to MongoDB:', pgError); - } + profilePicture: msg.profile_picture as string | undefined, + }, + username, + isBot, + ...withSelf(msg.user_id), + createdAt: msg.created_at as Date, + // Threading structure for agent readers (TASK-052's read half): + // the SELECT has carried these since #1106; this mapper was the + // last literal dropping them, which is why the cue teaching + // "continue in a thread" was held โ€” a peer agent reading context + // could not see that a thread existed. Explicit null = "not in a + // thread"; the Mongo fallback below omits the keys entirely = + // "this server cannot say" (same convention as the frontend). + reply_to_message_id: (msg.reply_to_message_id as string | number | null) ?? null, + thread_root_id: (msg.thread_root_id as string | number | null) ?? null, + author_kind: (msg.author_kind as 'kernel' | null) ?? null, + replyTo: (msg.replyTo as MessageNormalized['replyTo']) ?? null, + }; + }); + // Internal provenance for send-time read acknowledgement. A Mongo + // read never sees PG messages and must not clear a PG cursor. + Object.defineProperty(normalized, 'readSource', { value: 'postgres' }); + return normalized; } if (threadRootId) { diff --git a/backend/services/legacyMessageCopy.ts b/backend/services/legacyMessageCopy.ts new file mode 100644 index 0000000000..b267f67a4d --- /dev/null +++ b/backend/services/legacyMessageCopy.ts @@ -0,0 +1,124 @@ +/** + * Copies a MongoDB message into PostgreSQL under its Mongo id (TASK-503). + * + * PostgreSQL is the only message store once PG_HOST is set, so nothing writes + * Mongo messages there any more. What remains is history: rows written before + * an install gained Postgres, and the rows the old fallback wrote while PG was + * down. The operator script `backfill-local-messages-to-pg` moves them with + * `copyOne`. This module replaced the TASK-313 background reconciler, which + * moved only rows flagged `pgFallback` and so never moved the socket-path rows. + * + * - Idempotent. The PG insert is ON CONFLICT on legacy_mongo_id, so a second + * writer inserts nothing. The Mongo stamp is guarded on `reconciledAt` being + * absent, so a second stamp matches nothing. + * - The Mongo id keeps working. `PGMessage.resolveId` maps it to the PG row, + * so replies, thread roots, reactions and deletes that name it still land. + */ +interface CopyClient { + query: (sql: string, params?: unknown[]) => Promise<{ rows: Record[] }>; + release: () => void; +} + +// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require +const { pool } = require('../config/db-pg') as { + pool: { + connect: () => Promise; + query: (sql: string, params?: unknown[]) => Promise<{ rows: Record[] }>; + }; +}; +// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require +const MongoMessage = require('../models/Message') as { + updateOne: ( + filter: Record, + update: Record, + ) => Promise<{ matchedCount?: number; modifiedCount?: number }>; +}; + +export interface LegacyMessageDoc { + _id: { toString(): string }; + podId: { toString(): string }; + userId: { toString(): string }; + content: string; + messageType?: string; + authorKind?: 'kernel' | null; + payload?: unknown; + createdAt: Date; +} + +export const LEGACY_ID_INDEX = 'messages_legacy_mongo_id_key'; + +/** + * Builds the unique index ON CONFLICT needs, without blocking writes. + * + * CONCURRENTLY cannot run inside a transaction, so this issues the statement + * on its own autocommit connection. A failed concurrent build leaves an + * INVALID index behind; IF NOT EXISTS would keep that forever, so an invalid + * one is dropped and rebuilt. + */ +export async function ensureLegacyIdIndex(): Promise { + const client = await pool.connect(); + let locked = false; + try { + const lock = await client.query('SELECT pg_try_advisory_lock(313, 2) AS acquired'); + locked = lock.rows[0]?.acquired === true; + if (!locked) return false; + const state = await client.query( + `SELECT i.indisvalid FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relname = $1`, + [LEGACY_ID_INDEX], + ); + if (state.rows[0]?.indisvalid === false) { + await client.query(`DROP INDEX CONCURRENTLY IF EXISTS ${LEGACY_ID_INDEX}`); + } + await client.query( + `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ${LEGACY_ID_INDEX} + ON messages (legacy_mongo_id)`, + ); + return true; + } finally { + try { + if (locked) await client.query('SELECT pg_advisory_unlock(313, 2)'); + } finally { + client.release(); + } + } +} + +export async function copyOne(doc: LegacyMessageDoc): Promise<'moved' | 'present'> { + const id = doc._id.toString(); + const inserted = await pool.query( + `INSERT INTO messages (pod_id, user_id, content, message_type, created_at, legacy_mongo_id, author_kind, payload) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (legacy_mongo_id) DO NOTHING + RETURNING id`, + [ + doc.podId.toString(), + doc.userId.toString(), + doc.content || '', + doc.messageType || 'text', + doc.createdAt, + id, + doc.authorKind === 'kernel' ? 'kernel' : null, + doc.payload == null ? null : JSON.stringify(doc.payload), + ], + ); + let pgId = inserted.rows[0]?.id; + if (pgId === undefined) { + const existing = await pool.query( + 'SELECT id FROM messages WHERE legacy_mongo_id = $1', + [id], + ); + pgId = existing.rows[0]?.id; + } + if (pgId == null || !Number.isSafeInteger(Number(pgId)) || Number(pgId) <= 0) { + throw new Error('Copied PG row disappeared before stamping'); + } + // Stamp only after Postgres has the row. A crash between the two leaves an + // unstamped Mongo row that the next copy turns into `present`, then stamps. + await MongoMessage.updateOne( + { _id: doc._id, reconciledAt: { $exists: false } }, + { $set: { reconciledAt: new Date(), reconciledPgId: Number(pgId) } }, + ); + return inserted.rows[0] ? 'moved' : 'present'; +} diff --git a/backend/services/pgMessageReconcileService.ts b/backend/services/pgMessageReconcileService.ts deleted file mode 100644 index 575c20a4de..0000000000 --- a/backend/services/pgMessageReconcileService.ts +++ /dev/null @@ -1,240 +0,0 @@ -/** - * TASK-313 part 2 โ€” moves Mongo fallback messages into PostgreSQL. - * - * While Postgres is down, createMessage writes non-replies to Mongo. Once it - * is healthy this pass copies each unmoved row into `messages`, stamping the - * Mongo row with the PG id it became. Three properties carry the design: - * - * - Idempotent across replicas. The PG insert is ON CONFLICT on - * legacy_mongo_id, so a second writer inserts nothing. The Mongo stamp is - * guarded on `reconciledAt` being absent, so a second stamp matches nothing. - * - Bounded. One pass reads at most `maxPerPass` rows, unattempted rows first, then the oldest deferred attempts. - * - Malformed rows are permanently refused. Recoverable failures (including - * missing PG pods) are deferred so later rows can progress, then retried. - * - * Reads never merge the two stores: chat reads Postgres when it is healthy and - * Mongo only when it is not. A reconciled Mongo row therefore cannot be - * returned alongside its PG copy in one response. - */ -interface ReconcileClient { - query: (sql: string, params?: unknown[]) => Promise<{ rows: Record[] }>; - release: () => void; -} - -// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require -const { pool } = require('../config/db-pg') as { - pool: { - connect: () => Promise; - query: (sql: string, params?: unknown[]) => Promise<{ rows: Record[] }>; - }; -}; -// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require -const { syncPodFromMongo } = require('./pgPodSyncService') as { - syncPodFromMongo: (podId: string, requestingUserId: string) => Promise; -}; -// eslint-disable-next-line @typescript-eslint/no-require-imports, global-require -const MongoMessage = require('../models/Message') as { - find: (q: Record) => { - sort: (s: Record) => { - limit: (n: number) => { lean: () => Promise }; - }; - }; - updateOne: ( - filter: Record, - update: Record, - ) => Promise<{ matchedCount?: number; modifiedCount?: number }>; -}; - -interface MongoFallbackDoc { - _id: { toString(): string }; - podId: { toString(): string }; - userId: { toString(): string }; - content: string; - messageType?: string; - authorKind?: 'kernel' | null; - payload?: unknown; - createdAt: Date; -} - -export interface ReconcilePassResult { - /** False when Postgres did not answer; the pass did nothing. */ - healthy: boolean; - /** Rows this pass inserted into Postgres. */ - moved: number; - /** Rows Postgres already had (a concurrent pass or an earlier crash). */ - alreadyPresent: number; - /** Rows that failed this pass; permanent refusals are also stamped. */ - skipped: number; -} - -export const LEGACY_ID_INDEX = 'messages_legacy_mongo_id_key'; -export const DEFAULT_MAX_PER_PASS = 200; - -// FK failures can recover after pod synchronization; retry after a backoff. -export const RETRY_DELAY_MS = 60_000; -const isPermanentRowError = (err: unknown): boolean => { - const code = (err as { code?: string }).code; - return err instanceof TypeError || Boolean(code && code !== '23503' && /^(22|23)/.test(code)); -}; - -/** - * Builds the unique index ON CONFLICT needs, without blocking writes. - * - * CONCURRENTLY cannot run inside a transaction, so this issues the statement - * on its own autocommit connection. A failed concurrent build leaves an - * INVALID index behind; IF NOT EXISTS would keep that forever, so an invalid - * one is dropped and rebuilt. - */ -export async function ensureLegacyIdIndex(): Promise { - const client = await pool.connect(); - let locked = false; - try { - const lock = await client.query('SELECT pg_try_advisory_lock(313, 2) AS acquired'); - locked = lock.rows[0]?.acquired === true; - if (!locked) return false; - const state = await client.query( - `SELECT i.indisvalid FROM pg_index i - JOIN pg_class c ON c.oid = i.indexrelid - WHERE c.relname = $1`, - [LEGACY_ID_INDEX], - ); - if (state.rows[0]?.indisvalid === false) { - await client.query(`DROP INDEX CONCURRENTLY IF EXISTS ${LEGACY_ID_INDEX}`); - } - await client.query( - `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ${LEGACY_ID_INDEX} - ON messages (legacy_mongo_id)`, - ); - return true; - } finally { - try { - if (locked) await client.query('SELECT pg_advisory_unlock(313, 2)'); - } finally { - client.release(); - } - } -} - -export async function copyOne(doc: MongoFallbackDoc): Promise<'moved' | 'present'> { - const id = doc._id.toString(); - const inserted = await pool.query( - `INSERT INTO messages (pod_id, user_id, content, message_type, created_at, legacy_mongo_id, author_kind, payload) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (legacy_mongo_id) DO NOTHING - RETURNING id`, - [ - doc.podId.toString(), - doc.userId.toString(), - doc.content || '', - doc.messageType || 'text', - doc.createdAt, - id, - doc.authorKind === 'kernel' ? 'kernel' : null, - doc.payload == null ? null : JSON.stringify(doc.payload), - ], - ); - let pgId = inserted.rows[0]?.id; - if (pgId === undefined) { - const existing = await pool.query( - 'SELECT id FROM messages WHERE legacy_mongo_id = $1', - [id], - ); - pgId = existing.rows[0]?.id; - } - if (pgId == null || !Number.isSafeInteger(Number(pgId)) || Number(pgId) <= 0) { - throw new Error('Reconciled PG row disappeared before stamping'); - } - // Stamp only after Postgres has the row. A crash between the two leaves an - // unstamped Mongo row that the next pass turns into `present`, then stamps. - await MongoMessage.updateOne( - { _id: doc._id, reconciledAt: { $exists: false } }, - { $set: { reconciledAt: new Date(), reconciledPgId: Number(pgId) } }, - ); - return inserted.rows[0] ? 'moved' : 'present'; -} - -export async function reconcileMongoFallbackMessages( - { maxPerPass = DEFAULT_MAX_PER_PASS }: { maxPerPass?: number } = {}, -): Promise { - const result: ReconcilePassResult = { - healthy: false, moved: 0, alreadyPresent: 0, skipped: 0, - }; - try { - await pool.query('SELECT 1'); - } catch { - return result; - } - result.healthy = true; - - if (!await ensureLegacyIdIndex()) return result; - - const batch = await MongoMessage.find({ - pgFallback: true, - reconciledAt: { $exists: false }, - reconcileFailedAt: { $exists: false }, - $or: [{ reconcileRetryAt: { $exists: false } }, { reconcileRetryAt: { $lte: new Date() } }], - }) - .sort({ reconcileRetryAt: 1, createdAt: 1, _id: 1 }) - .limit(Math.max(1, Math.trunc(maxPerPass))) - .lean(); - - for (const doc of batch) { - try { - const outcome = await copyOne(doc).catch(async (err: { code?: string }) => { - if (err.code !== '23503') throw err; - // A Mongo-only pod can be mirrored once PG returns. Keep failures - // retryable; even a missing Mongo pod may be restored later. - try { - await syncPodFromMongo(doc.podId.toString(), doc.userId.toString()); - } catch { - throw err; - } - return copyOne(doc); - }); - if (outcome === 'moved') result.moved += 1; - else result.alreadyPresent += 1; - } catch (err) { - const e = err as { code?: string; message?: string }; - result.skipped += 1; - if (isPermanentRowError(err)) { - await MongoMessage.updateOne( - { _id: doc._id, reconcileFailedAt: { $exists: false } }, - { $set: { reconcileFailedAt: new Date(), reconcileError: e.message || e.code } }, - ); - } - else { - await MongoMessage.updateOne( - { _id: doc._id, reconciledAt: { $exists: false } }, - { $set: { reconcileRetryAt: new Date(Date.now() + RETRY_DELAY_MS) } }, - ); - } - console.warn('[pgMessageReconcile] row not moved:', doc._id.toString(), e.message); - } - } - return result; -} - -export default { reconcileMongoFallbackMessages, ensureLegacyIdIndex }; - -// One pass a minute per process. Replicas may each run one: the database -// (ON CONFLICT) and the Mongo stamp (set-once) make overlapping passes harmless. -// The in-process flag only keeps a slow pass from piling up behind itself. -let timer: ReturnType | null = null; -let passInFlight = false; - -export function initPgMessageReconcile(intervalMs = 60_000): void { - if (timer) return; - timer = setInterval(() => { - if (passInFlight) return; - passInFlight = true; - reconcileMongoFallbackMessages() - .then((r) => { - if (r.moved || r.skipped) { - console.log(`[pgMessageReconcile] moved=${r.moved} present=${r.alreadyPresent} skipped=${r.skipped}`); - } - }) - .catch((err: Error) => console.warn('[pgMessageReconcile] pass failed:', err.message)) - .finally(() => { passInFlight = false; }); - }, intervalMs); - timer.unref?.(); -} diff --git a/docs/architecture/long-term-review-2026-10.md b/docs/architecture/long-term-review-2026-10.md index 6eccdb79b7..1cbfac5713 100644 --- a/docs/architecture/long-term-review-2026-10.md +++ b/docs/architecture/long-term-review-2026-10.md @@ -390,7 +390,9 @@ Items are ranked by measured cost removed per day of effort. **Status (2026-10-30, TASK-503):** the trigger fired. SRE counted 13 MongoDB message rows created in the 30 days to 10-30, so this moved to Do now. - *Writes, done in the TASK-503 write PR:* once `PG_HOST` is set, the HTTP send, the socket send (`services/socketMessageWrite.ts`) and agent posts write PostgreSQL only. A failed write returns `MESSAGE_STORE_UNAVAILABLE` (503, or a socket `error`) instead of writing MongoDB. - *Where the rows came from (not yet confirmed per row):* there are three MongoDB writers. The socket wrote MongoDB whenever PostgreSQL was configured but had not finished booting, which happens on every rollout until `onMounted`. It set no `pgFallback` flag, so the reconciler never moved those rows and pod chat, which reads PostgreSQL, never showed them. The controller and agent fallbacks set `pgFallback: true`, and agent rows also carry `metadata`. Grouping the 13 rows by `pgFallback`, `metadata` and `reconciledAt` will tell the writers apart. -- *Still open:* removing the read, get-by-id and delete fallbacks, `pgMessageReconcileService`, and the MongoDB write used when no PostgreSQL is configured (the unit harness runs that way). That waits until the existing fallback rows are copied into PostgreSQL. +- *The 13 rows:* 4 carry `pgFallback`, none was reconciled. The reconciler moved only flagged rows, so the 9 socket rows would never have drained. An operator copies all 13 with `backfill-local-messages-to-pg --since` (dry run, then `--apply --force`). +- *Reads, delete and the reconciler, done in the TASK-503 reconciler-removal PR (merged after that copy):* once `PG_HOST` is set, the pod read (`repositories/messageRepository.ts`), the agent recent-message and author reads, and delete use PostgreSQL only, and a PG failure is an error or a 503, never a MongoDB read. `pgMessageReconcileService` is gone; its `copyOne` lives in `services/legacyMessageCopy.ts` for the backfill script. A Mongo id still resolves to its copied PG row through `legacy_mongo_id`. +- *Still open:* the MongoDB path used when no PostgreSQL is configured (the unit harness runs that way), and the MongoDB-only message readers (stats, activity, attention, approvals). Those go with the v4 data-model design. ### D. Remove OpenClaw, per ADR-021 Phases 3โ€“4 (3โ€“5 days, plus 1 for docs) diff --git a/docs/plans/pod-playbooks.md b/docs/plans/pod-playbooks.md index 2754deadb1..47345c480c 100644 --- a/docs/plans/pod-playbooks.md +++ b/docs/plans/pod-playbooks.md @@ -55,7 +55,7 @@ A `tool_call` card's decider is the person its proposer names; every other card **Two bindings.** `act: 'kernel'`, labelled "Runs when approved", makes the step `guard: 'kernel'`: the kernel performs the effect, with the ruling id on its receipt, and completes the row, so the person is asked once. Slices 1โ€“3 use effect `none`, where approve and complete are one write. Slice 4 adds `post`, `handoff` (ยง5) and `connector`, which parks the existing `tool_call` card for the connection's owner; that card's expiry is terminal (ADR-017:207โ€“212) and counts as a decline. `act: 'actor'`, labelled "Enforced on the record, not on the action", opens the row to its actor after Approve, and the done check refuses a run whose pointed step lacks an approved ruling. -**Each effect runs once.** On approve the ruling's CAS comes first, then the effect, then the complete. A crash can fall between the effect and the complete, and during a rollout two replicas' sweeps can retry the same step at once (`schedulerService.ts:492โ€“496`), so a retry can meet an effect that already ran. Each slice 4 effect is keyed by its card's `approvalId` at its target. A `handoff` run takes it as its trigger key, so the pair index refuses a second run row. A `connector` card stores it under a new sparse unique index on `ApprovalAction`, which has no unique index today (`ApprovalAction.ts:124โ€“125`). A `post` stores it in a new nullable `messages` column under a unique index, built as the legacy-id index is (`pgMessageReconcileService.ts:88โ€“107`), and fails rather than fall back to Mongo, as a thread-scoped post does (`agentMessageService.ts:1798โ€“1803`). A retry that meets the key takes the first effect as its own: a `post` or `handoff` row completes, and a `connector` row takes the state of the card it found. The receipt cannot guard a retry, because the crash can come before it is written. A `connector` row follows its card's ending, applied by a CAS on the open row found through `ruling.approvalId`, both beside the event that ending already enqueues (`approvalActionService.ts:719`) and when a retry finds the card ended: `ran` completes it; `declined`, `expired`, `failed` or `refused` blocks it and the run as a decline does, naming the ending, and so does a `moot` card, whose render never posted (`:519โ€“534`). A card still `flagged` leaves it waiting. +**Each effect runs once.** On approve the ruling's CAS comes first, then the effect, then the complete. A crash can fall between the effect and the complete, and during a rollout two replicas' sweeps can retry the same step at once (`schedulerService.ts:492โ€“496`), so a retry can meet an effect that already ran. Each slice 4 effect is keyed by its card's `approvalId` at its target. A `handoff` run takes it as its trigger key, so the pair index refuses a second run row. A `connector` card stores it under a new sparse unique index on `ApprovalAction`, which has no unique index today (`ApprovalAction.ts:124โ€“125`). A `post` stores it in a new nullable `messages` column under a unique index, built as the legacy-id index is (`services/legacyMessageCopy.ts`, `ensureLegacyIdIndex`), and fails rather than fall back to Mongo, as a thread-scoped post does (`agentMessageService.ts:1798โ€“1803`). A retry that meets the key takes the first effect as its own: a `post` or `handoff` row completes, and a `connector` row takes the state of the card it found. The receipt cannot guard a retry, because the crash can come before it is written. A `connector` row follows its card's ending, applied by a CAS on the open row found through `ruling.approvalId`, both beside the event that ending already enqueues (`approvalActionService.ts:719`) and when a retry finds the card ended: `ran` completes it; `declined`, `expired`, `failed` or `refused` blocks it and the run as a decline does, naming the ending, and so does a `moot` card, whose render never posted (`:519โ€“534`). A card still `flagged` leaves it waiting. No point is implicit, because no step's agent can write to a connector: in a grant pod a native turn can only read context, post (`nativeRuntimeService.ts:546`, `:613`, `:1999โ€“2003`) and call the grant's read tools (`grantBrokerProjectionService.ts:180`), and no BYO seat holds a grant today (ยง5). From 2f54e1e7704e0158a2fb35664c7d6cb0edc7efb9 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 10:14:34 -0700 Subject: [PATCH 2/4] test(showcase): a PG outage surfaces instead of falling back to Mongo (TASK-503) Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- backend/__tests__/service/showcase.test.js | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/__tests__/service/showcase.test.js b/backend/__tests__/service/showcase.test.js index 329b82e77f..4576514a0c 100644 --- a/backend/__tests__/service/showcase.test.js +++ b/backend/__tests__/service/showcase.test.js @@ -226,17 +226,8 @@ describe('Showcase (public read-only) routes', () => { }); describe('GET /api/showcase/:podId/messages', () => { - it.each(['primary', 'Mongo fallback'])('returns filtered, whitelisted messages for a public pod (anon): %s', async (store) => { - // Fault injection exercises the real Mongo fallback, not a mocked feed. - const failure = store === 'Mongo fallback' - ? jest.spyOn(PGMessage, 'findByPodId').mockRejectedValueOnce(new Error('injected PG outage')) - : null; - let res; - try { - res = await request(app).get(`/api/showcase/${publicPod._id}/messages`); - } finally { - failure?.mockRestore(); - } + it('returns filtered, whitelisted messages for a public pod (anon)', async () => { + const res = await request(app).get(`/api/showcase/${publicPod._id}/messages`); expect(res.status).toBe(200); expect(res.body).toHaveProperty('hasMore'); const contents = res.body.messages.map((m) => m.content); @@ -254,6 +245,21 @@ describe('Showcase (public read-only) routes', () => { expect(botMsg.author.displayName).toBe('Pixel'); }); + (REAL_DATABASES ? it : it.skip)('surfaces a PG outage instead of serving Mongo history', async () => { + const failure = jest.spyOn(PGMessage, 'findByPodId').mockRejectedValueOnce(new Error('injected PG outage')); + const mongoRead = jest.spyOn(Message, 'find'); + let res; + try { + res = await request(app).get(`/api/showcase/${publicPod._id}/messages`); + } finally { + failure.mockRestore(); + } + expect(res.status).toBe(500); + expect(res.body.messages).toBeUndefined(); + expect(mongoRead).not.toHaveBeenCalled(); + mongoRead.mockRestore(); + }); + (REAL_DATABASES ? it : it.skip)('does not substitute Mongo history for a successful empty PG read', async () => { const emptyPrimaryPod = await Pod.create({ name: 'Empty primary', type: 'team', createdBy: humanUser._id, From acbfe79530c7a9bccf24446c4daeec8b9b5fcb10 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:11:54 -0700 Subject: [PATCH 3/4] fix(messages): a PG read failure is a retryable 503, and one unreadable pod no longer stops the wake sweep (TASK-503) Keel and Sable HOLD at 2f54e1e7: - getRecentMessages and authorIdsSince now rethrow a PG failure as MESSAGE_STORE_UNAVAILABLE 503, the write's own error, so an agent post whose pre-write run-cap or dedupe read fails answers 503 instead of a 500 carrying the driver's text (which can name the DB host and port). - The assigned-task wake sweep catches a failed floor read per install, skips that install (fail closed, like humanActedInPodSince) and keeps going. Each new test fails without its fix. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- .../service/assignedTaskWakeService.test.js | 30 +++++++++++++++++++ ...agentMessageService.authorIdsSince.test.js | 3 +- .../agentMessageService.threadReads.test.js | 7 +++-- ...MessageService.threadRootBroadcast.test.js | 22 ++++++++++++++ backend/services/agentMessageService.ts | 24 ++++++++++----- backend/services/assignedTaskWakeService.ts | 14 +++++++-- backend/services/nativeRuntimeService.ts | 2 +- 7 files changed, 88 insertions(+), 14 deletions(-) diff --git a/backend/__tests__/service/assignedTaskWakeService.test.js b/backend/__tests__/service/assignedTaskWakeService.test.js index 7d10e3d359..c3469fcef5 100644 --- a/backend/__tests__/service/assignedTaskWakeService.test.js +++ b/backend/__tests__/service/assignedTaskWakeService.test.js @@ -245,3 +245,33 @@ it('does not cancel a reserved continuation when a skip arrives during preflight spy.mockRestore(); } }); + +it('skips an install whose floor read fails and still wakes the next one (TASK-503)', async () => { + // eslint-disable-next-line global-require + const AgentMessageService = require('../../services/agentMessageService'); + const otherPod = new mongoose.Types.ObjectId(); + const other = await User.create({ + username: 'loom-wake-test', email: 'loom-wake@example.test', password: 'test-password', + isBot: true, botType: 'agent', botMetadata: { agentName: 'codex', instanceId: 'loom' }, + }); + await AgentInstallation.create({ + agentName: 'codex', instanceId: 'loom', podId: otherPod, version: '1', installedBy: other._id, + config: { boardWake: { enabled: false } }, status: 'active', + }); + await Task.create({ podId: otherPod, taskNum: 1, taskId: 'TASK-9', title: 'Other seat', assignee: 'loom' }); + await completed(); + await completed({ instanceId: 'loom', podId: otherPod }); + // Whichever install the sweep reaches first has an unreadable pod. + const floor = jest.spyOn(AgentMessageService, 'holdsFloor') + .mockRejectedValueOnce(Object.assign(new Error('Messages are temporarily unavailable.'), { + code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503, + })) + .mockResolvedValue(false); + try { + expect(await dispatchAssignedTaskWakes(NOW)).toEqual({ eligible: 1, woken: 1 }); + expect(floor).toHaveBeenCalledTimes(2); + expect(AgentEventService.enqueue).toHaveBeenCalledTimes(1); + } finally { + floor.mockRestore(); + } +}); diff --git a/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js b/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js index 21bfbad6ee..f9e0baabd3 100644 --- a/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js +++ b/backend/__tests__/unit/services/agentMessageService.authorIdsSince.test.js @@ -37,7 +37,8 @@ describe('authorIdsSince', () => { process.env.PG_HOST = 'localhost'; PGMessage.findAuthorIdsSince.mockRejectedValue(new Error('pg down')); Message.distinct.mockResolvedValue(['person-2']); - await expect(AgentMessageService.authorIdsSince('pod-1', SINCE)).rejects.toThrow('pg down'); + await expect(AgentMessageService.authorIdsSince('pod-1', SINCE)) + .rejects.toMatchObject({ code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503 }); expect(Message.distinct).not.toHaveBeenCalled(); }); diff --git a/backend/__tests__/unit/services/agentMessageService.threadReads.test.js b/backend/__tests__/unit/services/agentMessageService.threadReads.test.js index a097d4a7a0..28d390cc84 100644 --- a/backend/__tests__/unit/services/agentMessageService.threadReads.test.js +++ b/backend/__tests__/unit/services/agentMessageService.threadReads.test.js @@ -89,8 +89,11 @@ describe('getRecentMessages โ€” thread structure on the PG path', () => { await expect( AgentMessageService.getRecentMessages(POD, 20, undefined, undefined, '57577'), - ).rejects.toThrow('pg down'); - await expect(AgentMessageService.getRecentMessages(POD, 20)).rejects.toThrow('pg down'); + ).rejects.toMatchObject({ code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503 }); + // The retryable error never carries the driver's text (it can name the DB host). + const failure = await AgentMessageService.getRecentMessages(POD, 20).catch((error) => error); + expect(failure).toMatchObject({ code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503 }); + expect(failure.message).not.toContain('pg down'); }); test('the Mongo store refuses a thread-scoped read instead of faking it', async () => { diff --git a/backend/__tests__/unit/services/agentMessageService.threadRootBroadcast.test.js b/backend/__tests__/unit/services/agentMessageService.threadRootBroadcast.test.js index 25d6e0499b..a2923cecd5 100644 --- a/backend/__tests__/unit/services/agentMessageService.threadRootBroadcast.test.js +++ b/backend/__tests__/unit/services/agentMessageService.threadRootBroadcast.test.js @@ -88,6 +88,7 @@ jest.mock('../../../models/File', () => ({ jest.mock('../../../models/pg/Message', () => ({ create: jest.fn(), findById: jest.fn(), + findByPodId: jest.fn(), })); const POD = '6a0da39bae757028b39f87a6'; @@ -189,6 +190,27 @@ describe('a configured PostgreSQL store is the only write path (TASK-503)', () = expect(Message).not.toHaveBeenCalled(); expect(emitted.find((e) => e.event === 'newMessage')).toBeUndefined(); }); + + it('fails a group-pod post with the same 503 when the pre-write run-cap read throws', async () => { + // The run-cap and dedupe reads run before the write. With no Mongo + // fallback left, the read's raw driver error used to escape as a 500 whose + // body carried the database host and port. + AgentMessageService.getRecentMessages.mockRestore(); + PGMessage.findByPodId.mockRejectedValue(new Error('connect ECONNREFUSED 10.9.8.7:5432')); + + const failure = await AgentMessageService.postMessage({ + agentName: 'sprint-review', + instanceId: 'default', + podId: POD, + content: 'A live agent message', + }).catch((error) => error); + + expect(failure).toMatchObject({ code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503 }); + expect(failure.message).not.toMatch(/10\.9\.8\.7|5432|ECONNREFUSED/); + expect(PGMessage.findByPodId).toHaveBeenCalled(); + expect(PGMessage.create).not.toHaveBeenCalled(); + expect(Message).not.toHaveBeenCalled(); + }); }); describe('agent broadcast carries author identity', () => { diff --git a/backend/services/agentMessageService.ts b/backend/services/agentMessageService.ts index c63169ceb0..5ad2b4aafd 100644 --- a/backend/services/agentMessageService.ts +++ b/backend/services/agentMessageService.ts @@ -60,6 +60,17 @@ const ATTACH_CLAIM_SCAN_LIMIT = 2000; // add an entry here only after observing it as a wrapper artifact in production. const BARE_RUNTIME_ARTIFACTS = new Set(['RGCTX']); +// PostgreSQL is the only message store once configured (TASK-503). Any failed +// read or write on it surfaces as this retryable error, never the driver's own +// message, which can carry the database host and port. +const messageStoreUnavailable = (op: string, cause: unknown): Error & { code: string; statusCode: number } => { + console.error(`[agent-msg] PostgreSQL message ${op} failed:`, (cause as Error)?.message); + return Object.assign( + new Error('Messages are temporarily unavailable. Please try again shortly.'), + { code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503, cause }, + ); +}; + const NO_REPLY_SENTINEL = 'NO_REPLY'; // `trim()` does not remove every invisible Unicode code point. Normalize a // decision-only copy before total-match and leading-sentinel checks so one @@ -1817,11 +1828,7 @@ class AgentMessageService { // PostgreSQL is the only message store once it is configured // (TASK-503). A failed write fails the post so the caller can retry; // a MongoDB row here would be invisible to pod chat, which reads PG. - console.error('[agent-msg] PostgreSQL message write failed:', (pgError as Error).message); - throw Object.assign( - new Error('Messages are temporarily unavailable. Please try again shortly.'), - { code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503, cause: pgError }, - ); + throw messageStoreUnavailable('write', pgError); } } @@ -2217,7 +2224,8 @@ class AgentMessageService { // caller fails closed rather than reading a store that has no new posts. if (PGMessage && process.env.PG_HOST) { const ids = await (PGMessage as { findAuthorIdsSince(id: unknown, at: Date): Promise }) - .findAuthorIdsSince(podId, since); + .findAuthorIdsSince(podId, since) + .catch((pgError: unknown) => { throw messageStoreUnavailable('author read', pgError); }); return [...new Set(ids.map(String))]; } const ids = await Message.distinct('userId', { @@ -2266,7 +2274,7 @@ class AgentMessageService { limit, before?.toISOString() ?? null, threadRootId ?? null, - ); + ).catch((pgError: unknown) => { throw messageStoreUnavailable('read', pgError); }); const normalized = messages.map((msg) => { const username = (msg.username as string) || 'Unknown'; @@ -2295,7 +2303,7 @@ class AgentMessageService { // last literal dropping them, which is why the cue teaching // "continue in a thread" was held โ€” a peer agent reading context // could not see that a thread existed. Explicit null = "not in a - // thread"; the Mongo fallback below omits the keys entirely = + // thread"; the no-PG Mongo read below omits the keys entirely = // "this server cannot say" (same convention as the frontend). reply_to_message_id: (msg.reply_to_message_id as string | number | null) ?? null, thread_root_id: (msg.thread_root_id as string | number | null) ?? null, diff --git a/backend/services/assignedTaskWakeService.ts b/backend/services/assignedTaskWakeService.ts index 11b81963e7..706204edb5 100644 --- a/backend/services/assignedTaskWakeService.ts +++ b/backend/services/assignedTaskWakeService.ts @@ -331,8 +331,18 @@ export async function dispatchAssignedTaskWakes(now = new Date()): Promise<{ eli logSkip(install, owned, 'other-work', now); continue; } - // eslint-disable-next-line no-await-in-loop - if (await holdsFloor(install, principalId)) { + let floor: boolean; + try { + // eslint-disable-next-line no-await-in-loop + floor = await holdsFloor(install, principalId); + } catch (error) { + // Fails closed for this install only, like humanActedInPodSince: an + // unreadable pod skips its own wake and the sweep moves on to the rest. + console.warn('[assigned-task-wake] floor unreadable:', String(install.podId), (error as Error).message); + logSkip(install, owned, 'floor-unreadable', now); + continue; + } + if (floor) { logSkip(install, owned, 'floor', now); continue; } diff --git a/backend/services/nativeRuntimeService.ts b/backend/services/nativeRuntimeService.ts index 5b9fa75438..6253036f37 100644 --- a/backend/services/nativeRuntimeService.ts +++ b/backend/services/nativeRuntimeService.ts @@ -1629,7 +1629,7 @@ export const resolveSeatUserId = async ( }; /** - * The pod's most recent messages, oldest first. PG first, Mongo fallback โ€” the + * The pod's most recent messages, oldest first, from the one configured store โ€” the * single read behind both `commonly_read_context` and the inline recent-messages * block, so the two can never disagree about what the room said. */ From 6abfe4db295347726dc30854931f3eb088afcf76 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:34:47 -0700 Subject: [PATCH 4/4] fix(messages): the pod-message repository read fails with the typed 503 too (TASK-503) Sable HOLD at acbfe795: listPodMessages' PG branch still threw the driver's text, which reaches agents through fetchRecentPodMessages and commonly_read_context. messageStoreUnavailable moves to services/messageStoreError so the repository and agentMessageService share it. The repository test fails without the catch. Co-Authored-By: Claude Opus 5.5 Co-authored-by: Ember Agent-Model: claude/claude-opus-5-5/medium --- .../unit/repositories/messageRepository.test.js | 7 +++++-- backend/repositories/messageRepository.ts | 4 +++- backend/services/agentMessageService.ts | 12 ++---------- backend/services/messageStoreError.ts | 15 +++++++++++++++ 4 files changed, 25 insertions(+), 13 deletions(-) create mode 100644 backend/services/messageStoreError.ts diff --git a/backend/__tests__/unit/repositories/messageRepository.test.js b/backend/__tests__/unit/repositories/messageRepository.test.js index a46ec55216..1e5f90c4cc 100644 --- a/backend/__tests__/unit/repositories/messageRepository.test.js +++ b/backend/__tests__/unit/repositories/messageRepository.test.js @@ -41,9 +41,12 @@ describe('listPodMessages', () => { }); it('with PG configured, a PG failure is an error and MongoDB is never read', async () => { - PGMessage.findByPodId.mockRejectedValue(new Error('pg down')); + PGMessage.findByPodId.mockRejectedValue(new Error('connect ECONNREFUSED 10.9.8.7:5432')); mongoChain([{ _id: 'm1', podId: 'p1', userId: 'u1', content: 'stale', createdAt: new Date() }]); - await expect(listPodMessages('p1')).rejects.toThrow('pg down'); + const failure = await listPodMessages('p1').catch((error) => error); + expect(failure).toMatchObject({ code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503 }); + // The driver text names the store's address; agent tool results echo err.message. + expect(failure.message).not.toMatch(/10\.9\.8\.7|5432|ECONNREFUSED/); expect(MongoMessage.find).not.toHaveBeenCalled(); }); diff --git a/backend/repositories/messageRepository.ts b/backend/repositories/messageRepository.ts index a1439325c0..eeedd82487 100644 --- a/backend/repositories/messageRepository.ts +++ b/backend/repositories/messageRepository.ts @@ -16,6 +16,7 @@ const PGMessage = require('../models/pg/Message'); // eslint-disable-next-line @typescript-eslint/no-require-imports const MongoMessage = require('../models/Message'); +import { messageStoreUnavailable } from '../services/messageStoreError'; export type PodMessageAuthor = { _id: string; username: string; profilePicture?: string; isBot?: boolean }; @@ -106,7 +107,8 @@ export async function listPodMessages( ): Promise { const safeLimit = clampLimit(limit); if (process.env.PG_HOST) { - return await PGMessage.findByPodId(podId, safeLimit, before || null) as PodMessage[]; + return await PGMessage.findByPodId(podId, safeLimit, before || null) + .catch((pgError: unknown) => { throw messageStoreUnavailable('read', pgError); }) as PodMessage[]; } const query: Record = { podId }; if (before) query.createdAt = { $lt: new Date(before) }; diff --git a/backend/services/agentMessageService.ts b/backend/services/agentMessageService.ts index 5ad2b4aafd..7fc05040e4 100644 --- a/backend/services/agentMessageService.ts +++ b/backend/services/agentMessageService.ts @@ -60,16 +60,8 @@ const ATTACH_CLAIM_SCAN_LIMIT = 2000; // add an entry here only after observing it as a wrapper artifact in production. const BARE_RUNTIME_ARTIFACTS = new Set(['RGCTX']); -// PostgreSQL is the only message store once configured (TASK-503). Any failed -// read or write on it surfaces as this retryable error, never the driver's own -// message, which can carry the database host and port. -const messageStoreUnavailable = (op: string, cause: unknown): Error & { code: string; statusCode: number } => { - console.error(`[agent-msg] PostgreSQL message ${op} failed:`, (cause as Error)?.message); - return Object.assign( - new Error('Messages are temporarily unavailable. Please try again shortly.'), - { code: 'MESSAGE_STORE_UNAVAILABLE', statusCode: 503, cause }, - ); -}; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { messageStoreUnavailable } = require('./messageStoreError'); const NO_REPLY_SENTINEL = 'NO_REPLY'; // `trim()` does not remove every invisible Unicode code point. Normalize a diff --git a/backend/services/messageStoreError.ts b/backend/services/messageStoreError.ts new file mode 100644 index 0000000000..cf4baf8c53 --- /dev/null +++ b/backend/services/messageStoreError.ts @@ -0,0 +1,15 @@ +/** + * PostgreSQL is the only message store once configured (TASK-503). Any failed + * read or write on it surfaces as this retryable error, never the driver's own + * message: that text can carry the database host and port, and agent tool + * results put an error's message straight into a model's context. + */ +export type MessageStoreUnavailable = Error & { code: 'MESSAGE_STORE_UNAVAILABLE'; statusCode: 503 }; + +export function messageStoreUnavailable(op: string, cause: unknown): MessageStoreUnavailable { + console.error(`[message-store] PostgreSQL message ${op} failed:`, (cause as Error)?.message); + return Object.assign( + new Error('Messages are temporarily unavailable. Please try again shortly.'), + { code: 'MESSAGE_STORE_UNAVAILABLE' as const, statusCode: 503 as const, cause }, + ); +}