feat(self-host): local profile runs PostgreSQL (TASK-503 slice 1)

README.md

@@ -93,7 +93,7 @@ cd commonly
 
 Then open http://localhost:3000.
 
-**Where it stands today:** this one-command install (`docker-compose.local.yml`) runs without Postgres. Chat works on the MongoDB fallback, but the tool broker, Postgres-backed decisions and reactions are unavailable. The full `docker-compose.yml` includes Postgres. More in the [self-hosting guide](docs/deployment/SELF_HOSTED.md).
+**Where it stands today:** this one-command install (`docker-compose.local.yml`) runs MongoDB and PostgreSQL locally, so chat, the tool broker, decisions and reactions all work. It is local-only: no TLS or public domain. More in the [self-hosting guide](docs/deployment/SELF_HOSTED.md).
 
 ## Docs
 

backend/__tests__/unit/deployment/localComposePostgres.test.js

@@ -0,0 +1,42 @@
+// TASK-503: chat messages become Postgres-only, so the one-command local
+// install (./install.sh -> docker-compose.local.yml) must run Postgres. It ran
+// MongoDB alone until this change, and removing the fallback would have broken
+// every local self-host. These are presence checks on the files the installer
+// actually reads; the boot itself is proven by running ./install.sh.
+const fs = require('fs');
+const path = require('path');
+
+const read = (name) => fs.readFileSync(path.join(__dirname, '../../../../', name), 'utf8');
+const compose = read('docker-compose.local.yml');
+const serviceBlock = (name) => {
+  const start = compose.search(new RegExp(`^  ${name}:\\s*$`, 'm'));
+  expect(start).toBeGreaterThanOrEqual(0);
+  const rest = compose.slice(start + 1);
+  const next = rest.search(/^ {2}[a-z][\w-]*:\s*$|^[a-z][\w-]*:\s*$/m);
+  return next === -1 ? rest : rest.slice(0, next);
+};
+
+describe('local compose runs Postgres for chat', () => {
+  it('declares a Postgres service pinned to a major version, with a health check and a volume', () => {
+    const pg = serviceBlock('postgres');
+    expect(pg).toMatch(/image: postgres:\d+(-alpine)?\s*$/m);
+    expect(pg).toMatch(/pg_isready/);
+    expect(pg).toMatch(/postgres-data:\/var\/lib\/postgresql\/data/);
+    expect(compose).toMatch(/^volumes:[\s\S]*^ {2}postgres-data:/m);
+    // Not published to the host: only the app ports are.
+    expect(pg).not.toMatch(/ports:/);
+  });
+
+  it('starts the backend after Postgres is healthy and hands it the PG_* settings', () => {
+    const backend = serviceBlock('backend');
+    expect(backend).toMatch(/postgres:\s*\n\s*condition: service_healthy/);
+    ['PG_HOST=postgres', 'PG_PORT=5432', 'PG_USER=', 'PG_PASSWORD=${PG_PASSWORD:?', 'PG_DATABASE=', 'PG_SSL_ENABLED=false']
+      .forEach((setting) => expect(backend).toContain(setting));
+  });
+
+  it.each(['install.sh', 'scripts/demo-bootstrap.sh'])('%s writes a PG_PASSWORD into .env when one is missing', (name) => {
+    const script = read(name);
+    expect(script).toMatch(/grep -Eq '\^PG_PASSWORD=\.\+\$'/);
+    expect(script).toMatch(/PG_PASSWORD=%s/);
+  });
+});

backend/__tests__/unit/scripts/backfillLocalMessagesToPg.test.js

@@ -0,0 +1,61 @@
+jest.mock('../../../services/pgMessageReconcileService', () => ({ copyOne: jest.fn(), ensureLegacyIdIndex: jest.fn() }));
+
+const { backfillLocalMessages } = require('../../../scripts/backfill-local-messages-to-pg');
+
+const doc = (id) => ({ _id: { toString: () => id }, podId: 'p', userId: 'u', content: id, createdAt: new Date('2026-01-01') });
+
+const fakeDeps = ({ pending = ['a1', 'a2', 'a3'], overlap = 0, copy } = {}) => {
+  const copied = [];
+  return {
+    copied,
+    countPending: jest.fn(async () => pending.length),
+    newestPendingAt: jest.fn(async () => new Date('2026-01-01')),
+    countOverlap: jest.fn(async () => overlap),
+    nextBatch: jest.fn(async (afterId, limit) => {
+      const start = afterId ? pending.indexOf(afterId) + 1 : 0;
+      return pending.slice(start, start + limit).map(doc);
+    }),
+    copy: jest.fn(copy || (async (d) => { copied.push(d._id.toString()); return 'moved'; })),
+  };
+};
+
+describe('backfillLocalMessages', () => {
+  it('a dry run reports the pending count and writes nothing', async () => {
+    const deps = fakeDeps();
+    const report = await backfillLocalMessages(deps);
+    expect(report).toMatchObject({ dryRun: true, pending: 3, refused: false, moved: 0 });
+    expect(deps.copy).not.toHaveBeenCalled();
+  });
+
+  it('--apply copies every pending message across batches', async () => {
+    const deps = fakeDeps({ pending: ['a1', 'a2', 'a3', 'a4', 'a5'] });
+    const report = await backfillLocalMessages(deps, { apply: true, batchSize: 2 });
+    expect(deps.copied).toEqual(['a1', 'a2', 'a3', 'a4', 'a5']);
+    expect(report).toMatchObject({ moved: 5, alreadyPresent: 0, failed: 0 });
+  });
+
+  it('counts rows PostgreSQL already had, so a rerun inserts nothing new', async () => {
+    const deps = fakeDeps({ copy: async () => 'present' });
+    const report = await backfillLocalMessages(deps, { apply: true });
+    expect(report).toMatchObject({ moved: 0, alreadyPresent: 3 });
+  });
+
+  it('refuses when PostgreSQL holds older messages that did not come from this MongoDB', async () => {
+    const deps = fakeDeps({ overlap: 12 });
+    const report = await backfillLocalMessages(deps, { apply: true });
+    expect(report).toMatchObject({ refused: true, overlap: 12, moved: 0 });
+    expect(deps.copy).not.toHaveBeenCalled();
+  });
+
+  it('--force copies despite the overlap', async () => {
+    const deps = fakeDeps({ overlap: 12 });
+    const report = await backfillLocalMessages(deps, { apply: true, force: true });
+    expect(report).toMatchObject({ refused: false, moved: 3 });
+  });
+
+  it('a failed row is counted and the run moves past it', async () => {
+    const deps = fakeDeps({ copy: async (d) => { if (d._id.toString() === 'a2') throw new Error('bad'); return 'moved'; } });
+    const report = await backfillLocalMessages(deps, { apply: true, batchSize: 1 });
+    expect(report).toMatchObject({ moved: 2, failed: 1 });
+  });
+});

backend/package.json

@@ -24,6 +24,7 @@
     "bootstrap:clawd-bot": "node scripts/bootstrap-clawd-bot.js",
     "backfill:attention-items": "ts-node scripts/backfill-attention-items.ts",
     "backfill:message-author-kind": "ts-node scripts/backfill-message-author-kind.ts",
+    "backfill:local-messages": "ts-node scripts/backfill-local-messages-to-pg.ts",
     "migrate:task-priority": "ts-node scripts/migrate-task-priority.ts",
     "migrate:task-source-ref-identity": "ts-node scripts/migrate-task-source-ref-identity.ts",
     "sweep:resolved-mention-attention": "ts-node scripts/sweep-resolved-mention-attention.ts",

backend/scripts/backfill-local-messages-to-pg.ts

@@ -0,0 +1,162 @@
+/**
+ * backfill-local-messages-to-pg — one-shot, for a self-hosted install that ran
+ * chat on MongoDB only and has just gained PostgreSQL (TASK-503).
+ *
+ * The local profile (docker-compose.local.yml) used to run without Postgres,
+ * so every chat message lived in Mongo. Once Postgres is healthy, chat reads
+ * Postgres only and never merges the two stores, so that history disappears
+ * from the UI. The background reconciler (pgMessageReconcileService) does not
+ * help: it moves only rows flagged `pgFallback`, written while PG was down.
+ *
+ * This copies every not-yet-reconciled Mongo message through the reconciler's
+ * own `copyOne`, so it inherits its guarantees: the PG insert is ON CONFLICT
+ * on `legacy_mongo_id` and the Mongo row is stamped `reconciledAt` afterwards,
+ * so a rerun or a crash mid-run inserts nothing twice.
+ *
+ * REFUSES on overlapping history. A deployment whose PG already holds messages
+ * that never came from this Mongo (no `legacy_mongo_id`) and that predate the
+ * newest Mongo row may have the same messages in both stores, copied by an
+ * older migration that did not record the Mongo id. Copying again would
+ * duplicate them. That is not the local-upgrade case, so it stops and says why;
+ * `--force` is for an operator who has checked.
+ *
+ * Dry run by default; `--apply` writes. Replies keep their text but not their
+ * reply edge, same as the reconciler.
+ *
+ *   npm run backfill:local-messages            # report only
+ *   npm run backfill:local-messages -- --apply
+ */
+/* eslint-disable no-console, no-await-in-loop */
+
+import mongoose from 'mongoose';
+import { copyOne, ensureLegacyIdIndex } from '../services/pgMessageReconcileService';
+
+type CopyResult = 'moved' | 'present';
+
+export interface PendingDoc {
+  _id: { toString(): string };
+  podId: { toString(): string };
+  userId: { toString(): string };
+  content: string;
+  createdAt: Date;
+}
+
+export interface LocalBackfillDeps {
+  countPending: () => Promise<number>;
+  newestPendingAt: () => Promise<Date | null>;
+  /** PG rows with no legacy_mongo_id written at or before `at`. */
+  countOverlap: (at: Date) => Promise<number>;
+  /** The next pending docs after `afterId`, oldest id first. */
+  nextBatch: (afterId: string | null, limit: number) => Promise<PendingDoc[]>;
+  copy: (doc: PendingDoc) => Promise<CopyResult>;
+}
+
+export interface LocalBackfillReport {
+  dryRun: boolean;
+  pending: number;
+  overlap: number;
+  refused: boolean;
+  moved: number;
+  alreadyPresent: number;
+  failed: number;
+}
+
+export async function backfillLocalMessages(
+  deps: LocalBackfillDeps,
+  { apply = false, force = false, batchSize = 500 }: { apply?: boolean; force?: boolean; batchSize?: number } = {},
+): Promise<LocalBackfillReport> {
+  const pending = await deps.countPending();
+  const newest = pending ? await deps.newestPendingAt() : null;
+  const overlap = newest ? await deps.countOverlap(newest) : 0;
+  const report: LocalBackfillReport = {
+    dryRun: !apply, pending, overlap, refused: overlap > 0 && !force, moved: 0, alreadyPresent: 0, failed: 0,
+  };
+  if (!apply || report.refused || !pending) return report;
+
+  let afterId: string | null = null;
+  for (;;) {
+    const batch = await deps.nextBatch(afterId, batchSize);
+    if (!batch.length) break;
+    for (const doc of batch) {
+      try {
+        if (await deps.copy(doc) === 'moved') report.moved += 1;
+        else report.alreadyPresent += 1;
+      } catch (err) {
+        report.failed += 1;
+        console.warn('[backfill-local-messages] row not copied:', doc._id.toString(), (err as Error).message);
+      }
+    }
+    afterId = batch[batch.length - 1]._id.toString();
+  }
+  return report;
+}
+
+interface PgPool {
+  query: (sql: string, values?: unknown[]) => Promise<{ rows: Array<Record<string, unknown>> }>;
+  end: () => Promise<void>;
+}
+
+const productionDeps = (pool: PgPool): LocalBackfillDeps => {
+  // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require
+  const Message = require('../models/Message');
+  // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require
+  const { syncPodFromMongo } = require('../services/pgPodSyncService');
+  const unreconciled = { reconciledAt: { $exists: false } };
+  return {
+    countPending: () => Message.countDocuments(unreconciled),
+    newestPendingAt: async () => {
+      const doc = await Message.findOne(unreconciled).sort({ createdAt: -1 }).select('createdAt').lean();
+      return doc?.createdAt || null;
+    },
+    countOverlap: async (at) => {
+      const { rows } = await pool.query(
+        'SELECT count(*)::int AS n FROM messages WHERE legacy_mongo_id IS NULL AND created_at <= $1',
+        [at],
+      );
+      return Number(rows[0]?.n || 0);
+    },
+    nextBatch: (afterId, limit) => Message.find(afterId
+      ? { ...unreconciled, _id: { $gt: new mongoose.Types.ObjectId(afterId) } }
+      : unreconciled).sort({ _id: 1 }).limit(limit).lean(),
+    // Same pod fallback as the reconciler: a Mongo-only pod is mirrored, then retried.
+    copy: (doc) => copyOne(doc).catch(async (err: { code?: string }) => {
+      if (err.code !== '23503') throw err;
+      await syncPodFromMongo(doc.podId.toString(), doc.userId.toString());
+      return copyOne(doc);
+    }),
+  };
+};
+
+export async function main(): Promise<void> {
+  const apply = process.argv.includes('--apply');
+  const force = process.argv.includes('--force');
+  if (!process.env.MONGO_URI) throw new Error('MONGO_URI is required');
+  // eslint-disable-next-line @typescript-eslint/no-require-imports, global-require
+  const { pool } = require('../config/db-pg') as { pool: PgPool | null };
+  if (!pool) throw new Error('PostgreSQL is not configured (PG_HOST unset)');
+
+  await mongoose.connect(process.env.MONGO_URI);
+  try {
+    if (!await ensureLegacyIdIndex()) throw new Error('could not ensure the legacy_mongo_id index');
+    const report = await backfillLocalMessages(productionDeps(pool), { apply, force });
+    console.log(`[backfill-local-messages] ${JSON.stringify(report)}`);
+    if (report.refused) {
+      console.log(`REFUSED — PostgreSQL already holds ${report.overlap} messages with no Mongo id`
+        + ' that predate the newest Mongo message. They may be the same history; copying would duplicate it.'
+        + ' Re-run with --force only if you have checked.');
+    } else if (!apply) {
+      console.log('DRY RUN — nothing written. Re-run with --apply.');
+    }
+    if (report.failed) process.exitCode = 1;
+  } finally {
+    await mongoose.disconnect();
+    await pool.end();
+  }
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error('local message backfill failed:', error);
+    process.exit(1);
+  });
+}

docker-compose.local.yml

@@ -1,10 +1,11 @@
 # Commonly — Local Install
-# Local, single-machine setup. MongoDB only, no cloud dependencies.
+# Local, single-machine setup. MongoDB + PostgreSQL, no cloud dependencies.
 #
 # Usage:
 #   ./install.sh
 #
-# `install.sh` creates an ignored .env with a strong JWT secret when needed.
+# `install.sh` creates an ignored .env with a strong JWT secret and Postgres
+# password when needed.
 # This profile binds the app to localhost-oriented URLs; do not use it as a
 # public-domain/TLS deployment.
 
@@ -18,6 +19,27 @@ x-logging: &default-logging
     max-file: "3"
 
 services:
+  # Chat messages live in Postgres (TASK-503). Pinned to a major so a fresh
+  # install never lands on a release whose data directory the volume can't read.
+  postgres:
+    logging: *default-logging
+    image: postgres:16-alpine
+    container_name: commonly-postgres
+    restart: unless-stopped
+    environment:
+      POSTGRES_USER: commonly
+      POSTGRES_PASSWORD: ${PG_PASSWORD:?Run ./install.sh or set PG_PASSWORD in .env before starting Commonly.}
+      POSTGRES_DB: commonly
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    networks:
+      - commonly
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U commonly -d commonly"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
   mongo:
     logging: *default-logging
     image: mongo:7
@@ -43,14 +65,23 @@ services:
     depends_on:
       mongo:
         condition: service_healthy
+      postgres:
+        condition: service_healthy
     ports:
       - "5000:5000"
     environment:
       - NODE_ENV=production
       - PORT=5000
 
-      # Database — MongoDB only, no PostgreSQL required
+      # Databases — MongoDB for documents, PostgreSQL for chat messages.
+      # The local Postgres has no TLS, so SSL is off; it is not published on a host port.
       - MONGO_URI=mongodb://mongo:27017/commonly
+      - PG_HOST=postgres
+      - PG_PORT=5432
+      - PG_USER=commonly
+      - PG_PASSWORD=${PG_PASSWORD:?Run ./install.sh or set PG_PASSWORD in .env before starting Commonly.}
+      - PG_DATABASE=commonly
+      - PG_SSL_ENABLED=false
 
       # Auth — install.sh creates JWT_SECRET in the ignored root .env file.
       - JWT_SECRET=${JWT_SECRET:?Run ./install.sh or set JWT_SECRET in .env before starting Commonly.}
@@ -102,3 +133,4 @@ networks:
 
 volumes:
   mongo-data:
+  postgres-data:

docs/deployment/SELF_HOSTED.md

@@ -10,17 +10,30 @@ matches where it will run:
 | Serve a public domain or run a multi-user production deployment | [Kubernetes deployment](./KUBERNETES.md) and the [Helm values reference](../self-hosting/helm-reference.md) |
 
 The Docker Compose profile is intentionally **local-only**. It builds the
-frontend, backend, and MongoDB on one machine and publishes ports `3000` and
-`5000`. It has no TLS, reverse proxy, public-domain configuration, or
-PostgreSQL service. Chat uses the MongoDB fallback. The tool broker,
-Postgres-backed decisions and reactions are unavailable in this profile;
-threaded agent messages are not enabled either. The tool broker explicitly
-requires PostgreSQL (`backend/models/ToolCall.ts`).
-
-The full [`docker-compose.yml`](../../docker-compose.yml) includes PostgreSQL;
-it requires its own environment configuration and is a separate profile.
-TASK-503 tracks adding PostgreSQL to the one-command local profile. Until that
-change lands, `./install.sh` continues to start the MongoDB-only profile.
+frontend and backend and runs MongoDB and PostgreSQL on one machine, publishing
+ports `3000` and `5000` only (neither database gets a host port). It has no
+TLS, reverse proxy or public-domain configuration. Chat messages, the tool
+broker (`backend/models/ToolCall.ts`), decisions and reactions use PostgreSQL,
+as in the full deployment. `./install.sh` generates the PostgreSQL password
+into the ignored `.env` beside the JWT secret, and adds one to an existing
+`.env` that predates it.
+
+**Upgrading a local install that ran before PostgreSQL was added.** Its chat
+history is in MongoDB, and chat now reads PostgreSQL, so the old messages
+won't show until they are copied. After `./install.sh` brings the stack up,
+check what would be copied, then copy it:
+
+```bash
+docker compose --env-file .env -f docker-compose.local.yml exec backend npm run backfill:local-messages
+docker compose --env-file .env -f docker-compose.local.yml exec backend npm run backfill:local-messages -- --apply
+```
+
+The copy is idempotent: each message is keyed by its MongoDB id, so a rerun
+copies nothing twice. It refuses when PostgreSQL already holds older messages
+that did not come from this MongoDB, since those may be the same history.
+
+The full [`docker-compose.yml`](../../docker-compose.yml) is a separate profile
+with its own environment configuration.
 Do not expose this Compose file directly to the internet.
 
 ## Prerequisites