diff --git a/README.md b/README.md index 77ca79bd2b..3484de307f 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ cd commonly Then open http://localhost:3000. -**Where it stands today:** this one-command install (`docker-compose.local.yml`) runs without Postgres. Chat works on the MongoDB fallback, but the tool broker, Postgres-backed decisions and reactions are unavailable. The full `docker-compose.yml` includes Postgres. More in the [self-hosting guide](docs/deployment/SELF_HOSTED.md). +**Where it stands today:** this one-command install (`docker-compose.local.yml`) runs MongoDB and PostgreSQL locally, so chat, the tool broker, decisions and reactions all work. It is local-only: no TLS or public domain. More in the [self-hosting guide](docs/deployment/SELF_HOSTED.md). ## Docs diff --git a/backend/__tests__/unit/deployment/localComposePostgres.test.js b/backend/__tests__/unit/deployment/localComposePostgres.test.js new file mode 100644 index 0000000000..a28b5a334e --- /dev/null +++ b/backend/__tests__/unit/deployment/localComposePostgres.test.js @@ -0,0 +1,42 @@ +// TASK-503: chat messages become Postgres-only, so the one-command local +// install (./install.sh -> docker-compose.local.yml) must run Postgres. It ran +// MongoDB alone until this change, and removing the fallback would have broken +// every local self-host. These are presence checks on the files the installer +// actually reads; the boot itself is proven by running ./install.sh. +const fs = require('fs'); +const path = require('path'); + +const read = (name) => fs.readFileSync(path.join(__dirname, '../../../../', name), 'utf8'); +const compose = read('docker-compose.local.yml'); +const serviceBlock = (name) => { + const start = compose.search(new RegExp(`^ ${name}:\\s*$`, 'm')); + expect(start).toBeGreaterThanOrEqual(0); + const rest = compose.slice(start + 1); + const next = rest.search(/^ {2}[a-z][\w-]*:\s*$|^[a-z][\w-]*:\s*$/m); + return next === -1 ? rest : rest.slice(0, next); +}; + +describe('local compose runs Postgres for chat', () => { + it('declares a Postgres service pinned to a major version, with a health check and a volume', () => { + const pg = serviceBlock('postgres'); + expect(pg).toMatch(/image: postgres:\d+(-alpine)?\s*$/m); + expect(pg).toMatch(/pg_isready/); + expect(pg).toMatch(/postgres-data:\/var\/lib\/postgresql\/data/); + expect(compose).toMatch(/^volumes:[\s\S]*^ {2}postgres-data:/m); + // Not published to the host: only the app ports are. + expect(pg).not.toMatch(/ports:/); + }); + + it('starts the backend after Postgres is healthy and hands it the PG_* settings', () => { + const backend = serviceBlock('backend'); + expect(backend).toMatch(/postgres:\s*\n\s*condition: service_healthy/); + ['PG_HOST=postgres', 'PG_PORT=5432', 'PG_USER=', 'PG_PASSWORD=${PG_PASSWORD:?', 'PG_DATABASE=', 'PG_SSL_ENABLED=false'] + .forEach((setting) => expect(backend).toContain(setting)); + }); + + it.each(['install.sh', 'scripts/demo-bootstrap.sh'])('%s writes a PG_PASSWORD into .env when one is missing', (name) => { + const script = read(name); + expect(script).toMatch(/grep -Eq '\^PG_PASSWORD=\.\+\$'/); + expect(script).toMatch(/PG_PASSWORD=%s/); + }); +}); diff --git a/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js new file mode 100644 index 0000000000..98959584f1 --- /dev/null +++ b/backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js @@ -0,0 +1,61 @@ +jest.mock('../../../services/pgMessageReconcileService', () => ({ copyOne: jest.fn(), ensureLegacyIdIndex: jest.fn() })); + +const { backfillLocalMessages } = require('../../../scripts/backfill-local-messages-to-pg'); + +const doc = (id) => ({ _id: { toString: () => id }, podId: 'p', userId: 'u', content: id, createdAt: new Date('2026-01-01') }); + +const fakeDeps = ({ pending = ['a1', 'a2', 'a3'], overlap = 0, copy } = {}) => { + const copied = []; + return { + copied, + countPending: jest.fn(async () => pending.length), + newestPendingAt: jest.fn(async () => new Date('2026-01-01')), + countOverlap: jest.fn(async () => overlap), + nextBatch: jest.fn(async (afterId, limit) => { + const start = afterId ? pending.indexOf(afterId) + 1 : 0; + return pending.slice(start, start + limit).map(doc); + }), + copy: jest.fn(copy || (async (d) => { copied.push(d._id.toString()); return 'moved'; })), + }; +}; + +describe('backfillLocalMessages', () => { + it('a dry run reports the pending count and writes nothing', async () => { + const deps = fakeDeps(); + const report = await backfillLocalMessages(deps); + expect(report).toMatchObject({ dryRun: true, pending: 3, refused: false, moved: 0 }); + expect(deps.copy).not.toHaveBeenCalled(); + }); + + it('--apply copies every pending message across batches', async () => { + const deps = fakeDeps({ pending: ['a1', 'a2', 'a3', 'a4', 'a5'] }); + const report = await backfillLocalMessages(deps, { apply: true, batchSize: 2 }); + expect(deps.copied).toEqual(['a1', 'a2', 'a3', 'a4', 'a5']); + expect(report).toMatchObject({ moved: 5, alreadyPresent: 0, failed: 0 }); + }); + + it('counts rows PostgreSQL already had, so a rerun inserts nothing new', async () => { + const deps = fakeDeps({ copy: async () => 'present' }); + const report = await backfillLocalMessages(deps, { apply: true }); + expect(report).toMatchObject({ moved: 0, alreadyPresent: 3 }); + }); + + it('refuses when PostgreSQL holds older messages that did not come from this MongoDB', async () => { + const deps = fakeDeps({ overlap: 12 }); + const report = await backfillLocalMessages(deps, { apply: true }); + expect(report).toMatchObject({ refused: true, overlap: 12, moved: 0 }); + expect(deps.copy).not.toHaveBeenCalled(); + }); + + it('--force copies despite the overlap', async () => { + const deps = fakeDeps({ overlap: 12 }); + const report = await backfillLocalMessages(deps, { apply: true, force: true }); + expect(report).toMatchObject({ refused: false, moved: 3 }); + }); + + it('a failed row is counted and the run moves past it', async () => { + const deps = fakeDeps({ copy: async (d) => { if (d._id.toString() === 'a2') throw new Error('bad'); return 'moved'; } }); + const report = await backfillLocalMessages(deps, { apply: true, batchSize: 1 }); + expect(report).toMatchObject({ moved: 2, failed: 1 }); + }); +}); diff --git a/backend/package.json b/backend/package.json index 876e03e633..53d36778c7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -24,6 +24,7 @@ "bootstrap:clawd-bot": "node scripts/bootstrap-clawd-bot.js", "backfill:attention-items": "ts-node scripts/backfill-attention-items.ts", "backfill:message-author-kind": "ts-node scripts/backfill-message-author-kind.ts", + "backfill:local-messages": "ts-node scripts/backfill-local-messages-to-pg.ts", "migrate:task-priority": "ts-node scripts/migrate-task-priority.ts", "migrate:task-source-ref-identity": "ts-node scripts/migrate-task-source-ref-identity.ts", "sweep:resolved-mention-attention": "ts-node scripts/sweep-resolved-mention-attention.ts", diff --git a/backend/scripts/backfill-local-messages-to-pg.ts b/backend/scripts/backfill-local-messages-to-pg.ts new file mode 100644 index 0000000000..7a8075c24f --- /dev/null +++ b/backend/scripts/backfill-local-messages-to-pg.ts @@ -0,0 +1,162 @@ +/** + * backfill-local-messages-to-pg — one-shot, for a self-hosted install that ran + * chat on MongoDB only and has just gained PostgreSQL (TASK-503). + * + * The local profile (docker-compose.local.yml) used to run without Postgres, + * so every chat message lived in Mongo. Once Postgres is healthy, chat reads + * Postgres only and never merges the two stores, so that history disappears + * from the UI. The background reconciler (pgMessageReconcileService) does not + * help: it moves only rows flagged `pgFallback`, written while PG was down. + * + * This copies every not-yet-reconciled Mongo message through the reconciler's + * own `copyOne`, so it inherits its guarantees: the PG insert is ON CONFLICT + * on `legacy_mongo_id` and the Mongo row is stamped `reconciledAt` afterwards, + * so a rerun or a crash mid-run inserts nothing twice. + * + * REFUSES on overlapping history. A deployment whose PG already holds messages + * that never came from this Mongo (no `legacy_mongo_id`) and that predate the + * newest Mongo row may have the same messages in both stores, copied by an + * older migration that did not record the Mongo id. Copying again would + * duplicate them. That is not the local-upgrade case, so it stops and says why; + * `--force` is for an operator who has checked. + * + * Dry run by default; `--apply` writes. Replies keep their text but not their + * reply edge, same as the reconciler. + * + * npm run backfill:local-messages # report only + * npm run backfill:local-messages -- --apply + */ +/* eslint-disable no-console, no-await-in-loop */ + +import mongoose from 'mongoose'; +import { copyOne, ensureLegacyIdIndex } from '../services/pgMessageReconcileService'; + +type CopyResult = 'moved' | 'present'; + +export interface PendingDoc { + _id: { toString(): string }; + podId: { toString(): string }; + userId: { toString(): string }; + content: string; + createdAt: Date; +} + +export interface LocalBackfillDeps { + countPending: () => Promise; + newestPendingAt: () => Promise; + /** PG rows with no legacy_mongo_id written at or before `at`. */ + countOverlap: (at: Date) => Promise; + /** The next pending docs after `afterId`, oldest id first. */ + nextBatch: (afterId: string | null, limit: number) => Promise; + copy: (doc: PendingDoc) => Promise; +} + +export interface LocalBackfillReport { + dryRun: boolean; + pending: number; + overlap: number; + refused: boolean; + moved: number; + alreadyPresent: number; + failed: number; +} + +export async function backfillLocalMessages( + deps: LocalBackfillDeps, + { apply = false, force = false, batchSize = 500 }: { apply?: boolean; force?: boolean; batchSize?: number } = {}, +): Promise { + const pending = await deps.countPending(); + const newest = pending ? await deps.newestPendingAt() : null; + const overlap = newest ? await deps.countOverlap(newest) : 0; + const report: LocalBackfillReport = { + dryRun: !apply, pending, overlap, refused: overlap > 0 && !force, moved: 0, alreadyPresent: 0, failed: 0, + }; + if (!apply || report.refused || !pending) return report; + + let afterId: string | null = null; + for (;;) { + const batch = await deps.nextBatch(afterId, batchSize); + if (!batch.length) break; + for (const doc of batch) { + try { + if (await deps.copy(doc) === 'moved') report.moved += 1; + else report.alreadyPresent += 1; + } catch (err) { + report.failed += 1; + console.warn('[backfill-local-messages] row not copied:', doc._id.toString(), (err as Error).message); + } + } + afterId = batch[batch.length - 1]._id.toString(); + } + return report; +} + +interface PgPool { + query: (sql: string, values?: unknown[]) => Promise<{ rows: Array> }>; + end: () => Promise; +} + +const productionDeps = (pool: PgPool): LocalBackfillDeps => { + // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require + const Message = require('../models/Message'); + // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require + const { syncPodFromMongo } = require('../services/pgPodSyncService'); + const unreconciled = { reconciledAt: { $exists: false } }; + return { + countPending: () => Message.countDocuments(unreconciled), + newestPendingAt: async () => { + const doc = await Message.findOne(unreconciled).sort({ createdAt: -1 }).select('createdAt').lean(); + return doc?.createdAt || null; + }, + countOverlap: async (at) => { + const { rows } = await pool.query( + 'SELECT count(*)::int AS n FROM messages WHERE legacy_mongo_id IS NULL AND created_at <= $1', + [at], + ); + return Number(rows[0]?.n || 0); + }, + nextBatch: (afterId, limit) => Message.find(afterId + ? { ...unreconciled, _id: { $gt: new mongoose.Types.ObjectId(afterId) } } + : unreconciled).sort({ _id: 1 }).limit(limit).lean(), + // Same pod fallback as the reconciler: a Mongo-only pod is mirrored, then retried. + copy: (doc) => copyOne(doc).catch(async (err: { code?: string }) => { + if (err.code !== '23503') throw err; + await syncPodFromMongo(doc.podId.toString(), doc.userId.toString()); + return copyOne(doc); + }), + }; +}; + +export async function main(): Promise { + const apply = process.argv.includes('--apply'); + const force = process.argv.includes('--force'); + if (!process.env.MONGO_URI) throw new Error('MONGO_URI is required'); + // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require + const { pool } = require('../config/db-pg') as { pool: PgPool | null }; + if (!pool) throw new Error('PostgreSQL is not configured (PG_HOST unset)'); + + await mongoose.connect(process.env.MONGO_URI); + try { + if (!await ensureLegacyIdIndex()) throw new Error('could not ensure the legacy_mongo_id index'); + const report = await backfillLocalMessages(productionDeps(pool), { apply, force }); + console.log(`[backfill-local-messages] ${JSON.stringify(report)}`); + if (report.refused) { + console.log(`REFUSED — PostgreSQL already holds ${report.overlap} messages with no Mongo id` + + ' that predate the newest Mongo message. They may be the same history; copying would duplicate it.' + + ' Re-run with --force only if you have checked.'); + } else if (!apply) { + console.log('DRY RUN — nothing written. Re-run with --apply.'); + } + if (report.failed) process.exitCode = 1; + } finally { + await mongoose.disconnect(); + await pool.end(); + } +} + +if (require.main === module) { + main().catch((error) => { + console.error('local message backfill failed:', error); + process.exit(1); + }); +} diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 2830796219..b2ac9a8a0d 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -1,10 +1,11 @@ # Commonly — Local Install -# Local, single-machine setup. MongoDB only, no cloud dependencies. +# Local, single-machine setup. MongoDB + PostgreSQL, no cloud dependencies. # # Usage: # ./install.sh # -# `install.sh` creates an ignored .env with a strong JWT secret when needed. +# `install.sh` creates an ignored .env with a strong JWT secret and Postgres +# password when needed. # This profile binds the app to localhost-oriented URLs; do not use it as a # public-domain/TLS deployment. @@ -18,6 +19,27 @@ x-logging: &default-logging max-file: "3" services: + # Chat messages live in Postgres (TASK-503). Pinned to a major so a fresh + # install never lands on a release whose data directory the volume can't read. + postgres: + logging: *default-logging + image: postgres:16-alpine + container_name: commonly-postgres + restart: unless-stopped + environment: + POSTGRES_USER: commonly + POSTGRES_PASSWORD: ${PG_PASSWORD:?Run ./install.sh or set PG_PASSWORD in .env before starting Commonly.} + POSTGRES_DB: commonly + volumes: + - postgres-data:/var/lib/postgresql/data + networks: + - commonly + healthcheck: + test: ["CMD-SHELL", "pg_isready -U commonly -d commonly"] + interval: 10s + timeout: 5s + retries: 5 + mongo: logging: *default-logging image: mongo:7 @@ -43,14 +65,23 @@ services: depends_on: mongo: condition: service_healthy + postgres: + condition: service_healthy ports: - "5000:5000" environment: - NODE_ENV=production - PORT=5000 - # Database — MongoDB only, no PostgreSQL required + # Databases — MongoDB for documents, PostgreSQL for chat messages. + # The local Postgres has no TLS, so SSL is off; it is not published on a host port. - MONGO_URI=mongodb://mongo:27017/commonly + - PG_HOST=postgres + - PG_PORT=5432 + - PG_USER=commonly + - PG_PASSWORD=${PG_PASSWORD:?Run ./install.sh or set PG_PASSWORD in .env before starting Commonly.} + - PG_DATABASE=commonly + - PG_SSL_ENABLED=false # Auth — install.sh creates JWT_SECRET in the ignored root .env file. - JWT_SECRET=${JWT_SECRET:?Run ./install.sh or set JWT_SECRET in .env before starting Commonly.} @@ -102,3 +133,4 @@ networks: volumes: mongo-data: + postgres-data: diff --git a/docs/deployment/SELF_HOSTED.md b/docs/deployment/SELF_HOSTED.md index 4e69fe2424..f39091e51d 100644 --- a/docs/deployment/SELF_HOSTED.md +++ b/docs/deployment/SELF_HOSTED.md @@ -10,17 +10,30 @@ matches where it will run: | Serve a public domain or run a multi-user production deployment | [Kubernetes deployment](./KUBERNETES.md) and the [Helm values reference](../self-hosting/helm-reference.md) | The Docker Compose profile is intentionally **local-only**. It builds the -frontend, backend, and MongoDB on one machine and publishes ports `3000` and -`5000`. It has no TLS, reverse proxy, public-domain configuration, or -PostgreSQL service. Chat uses the MongoDB fallback. The tool broker, -Postgres-backed decisions and reactions are unavailable in this profile; -threaded agent messages are not enabled either. The tool broker explicitly -requires PostgreSQL (`backend/models/ToolCall.ts`). - -The full [`docker-compose.yml`](../../docker-compose.yml) includes PostgreSQL; -it requires its own environment configuration and is a separate profile. -TASK-503 tracks adding PostgreSQL to the one-command local profile. Until that -change lands, `./install.sh` continues to start the MongoDB-only profile. +frontend and backend and runs MongoDB and PostgreSQL on one machine, publishing +ports `3000` and `5000` only (neither database gets a host port). It has no +TLS, reverse proxy or public-domain configuration. Chat messages, the tool +broker (`backend/models/ToolCall.ts`), decisions and reactions use PostgreSQL, +as in the full deployment. `./install.sh` generates the PostgreSQL password +into the ignored `.env` beside the JWT secret, and adds one to an existing +`.env` that predates it. + +**Upgrading a local install that ran before PostgreSQL was added.** Its chat +history is in MongoDB, and chat now reads PostgreSQL, so the old messages +won't show until they are copied. After `./install.sh` brings the stack up, +check what would be copied, then copy it: + +```bash +docker compose --env-file .env -f docker-compose.local.yml exec backend npm run backfill:local-messages +docker compose --env-file .env -f docker-compose.local.yml exec backend npm run backfill:local-messages -- --apply +``` + +The copy is idempotent: each message is keyed by its MongoDB id, so a rerun +copies nothing twice. It refuses when PostgreSQL already holds older messages +that did not come from this MongoDB, since those may be the same history. + +The full [`docker-compose.yml`](../../docker-compose.yml) is a separate profile +with its own environment configuration. Do not expose this Compose file directly to the internet. ## Prerequisites diff --git a/docs/self-hosting/helm-reference.md b/docs/self-hosting/helm-reference.md index cc979c9792..acacadba47 100644 --- a/docs/self-hosting/helm-reference.md +++ b/docs/self-hosting/helm-reference.md @@ -10,10 +10,9 @@ must override that value to `false`; the current hosted dev values do so. ## One-command local install -`./install.sh` uses `docker-compose.local.yml`, which currently runs without -Postgres. Chat uses the MongoDB fallback. The tool broker, Postgres-backed -decisions and reactions are unavailable. The full `docker-compose.yml` includes -Postgres; adding it to the local profile is tracked in TASK-503. +`./install.sh` uses `docker-compose.local.yml`, which runs MongoDB and a +pinned PostgreSQL 16 alongside the app, so chat, the tool broker, decisions +and reactions behave as they do under Helm. See the [local installation guide](../deployment/SELF_HOSTED.md) for profile selection. The Helm configuration below supports a PostgreSQL service or an external PostgreSQL database. diff --git a/install.sh b/install.sh index bea45e8c36..065ba68708 100755 --- a/install.sh +++ b/install.sh @@ -37,11 +37,19 @@ else echo " ✓ Created $ENV_FILE with a generated JWT secret" fi +# Chat messages live in Postgres (TASK-503). An .env from before it has no +# PG_PASSWORD, so add one rather than refusing to start. +if ! grep -Eq '^PG_PASSWORD=.+$' "$ENV_FILE"; then + if ! command -v openssl &>/dev/null; then + echo " ✗ openssl is required to generate a local PG_PASSWORD" + exit 1 + fi + ( umask 077; printf 'PG_PASSWORD=%s\n' "$(openssl rand -hex 24)" >> "$ENV_FILE" ) + echo " ✓ Added a generated Postgres password to $ENV_FILE" +fi + echo "" -echo " Local profile: docker-compose.local.yml (without Postgres)." -echo " Chat uses the MongoDB fallback. The tool broker, Postgres-backed" -echo " decisions and reactions are unavailable." -echo " docker-compose.yml includes Postgres; TASK-503 tracks adding it locally." +echo " Local profile: docker-compose.local.yml (MongoDB + PostgreSQL)." echo "" echo " Starting Commonly..." echo "" @@ -60,3 +68,6 @@ echo "" echo " Logs: $COMPOSE --env-file $ENV_FILE -f docker-compose.local.yml logs -f" echo " Stop: $COMPOSE --env-file $ENV_FILE -f docker-compose.local.yml down" echo "" +echo " Upgrading from a MongoDB-only install? Copy old chat history into PostgreSQL:" +echo " $COMPOSE --env-file $ENV_FILE -f docker-compose.local.yml exec backend npm run backfill:local-messages -- --apply" +echo "" diff --git a/scripts/demo-bootstrap.sh b/scripts/demo-bootstrap.sh index 114812fd7f..39ca8dd2a0 100755 --- a/scripts/demo-bootstrap.sh +++ b/scripts/demo-bootstrap.sh @@ -118,6 +118,11 @@ elif ! grep -Eq '^JWT_SECRET=.+$' "$ENV_FILE" \ err "$ENV_FILE needs a non-default JWT_SECRET. Set one, then run this command again." exit 1 fi +# The local profile runs Postgres (TASK-503); an older .env has no password for it. +if ! grep -Eq '^PG_PASSWORD=.+$' "$ENV_FILE"; then + ( umask 077; printf 'PG_PASSWORD=%s\n' "$(openssl rand -hex 24)" >> "$ENV_FILE" ) + ok "Added a generated Postgres password to .env" +fi # ── 2. Bring the stack up ─────────────────────────────────────────────────── section "Starting local Commonly stack"