Fix/1530 1531 1545 1546

.github/workflows/backend-governance.yml

@@ -85,3 +85,6 @@ jobs:
 
       - name: Check for database schema drift
         run: npm run db:check-drift
+
+      - name: Check dev (SQLite) vs prod (PostgreSQL) column type mapping
+        run: npm run db:check-types

backend/docs/DB_TYPE_MAPPING.md

@@ -0,0 +1,36 @@
+# Dev / prod column type mapping
+
+Development and tests run Prisma against SQLite (`prisma/schema.prisma`,
+`file:./dev.db`). Production storage is PostgreSQL, created by the raw SQL
+migrations in `migrations/*.sql`. SQLite has no exact decimal, JSON or
+timezone-aware timestamp type, so those columns use different types on each
+side:
+
+| PostgreSQL (prod)            | Prisma / SQLite (dev) | Encoding rule |
+|------------------------------|-----------------------|---------------|
+| `NUMERIC(38, 18)`            | `String`              | Plain decimal string, at most 20 integer and 18 fractional digits. Parse and do arithmetic with `NumericDecimal` and write with `toNumeric38_18` (`src/numericColumns.ts`); plain `Decimal` keeps only 20 significant digits and rounds silently. |
+| `NUMERIC(12, 6)`             | `String`              | Plain decimal string. |
+| `JSONB`                      | `String`              | `JSON.stringify` on write, `JSON.parse` on read. |
+| `TIMESTAMPTZ`, `DATE`        | `DateTime`            | Always UTC. |
+
+The column-by-column mapping lives in `prisma/postgres-type-mapping.json`.
+Every PostgreSQL `NUMERIC` / `JSONB` / `TIMESTAMPTZ` / `DATE` column must be
+listed there, either with the Prisma field that mirrors it or `null` when it
+has no dev counterpart.
+
+## Drift check
+
+```bash
+npm run db:check-types
+```
+
+`scripts/check-type-mapping.js` parses the schema, the migrations and the
+mapping (no database needed) and fails when:
+
+- a new `NUMERIC` / `JSONB` / `TIMESTAMPTZ` / `DATE` column is not in the mapping,
+- a mapped column or Prisma field no longer exists,
+- a mapped Prisma field's type differs from the mapping for its PostgreSQL type
+  (for example an amount changed to `Float`).
+
+It runs in the Backend Governance workflow next to `db:check-drift`, which
+checks the applied PostgreSQL migrations themselves.

backend/package.json

@@ -31,6 +31,7 @@
     "snapshots:check": "tsx scripts/check-schema-snapshots.ts",
     "db:migrate": "node scripts/migrate-postgres.js",
     "db:check-drift": "node scripts/check-postgres-drift.js",
+    "db:check-types": "node scripts/check-type-mapping.js",
     "adrs:check": "node scripts/check-adrs.js",
     "prisma:generate": "prisma generate --schema prisma/schema.prisma",
     "prisma:schema-check": "prisma validate --schema prisma/schema.prisma && prisma migrate diff --from-migrations prisma/migrations --to-schema-datamodel prisma/schema.prisma --exit-code",

backend/prisma/postgres-type-mapping.json

@@ -0,0 +1,29 @@
+{
+  "$comment": "Dev (Prisma/SQLite) <-> prod (migrations/*.sql, PostgreSQL) type mapping. Checked by scripts/check-type-mapping.js; see docs/DB_TYPE_MAPPING.md.",
+  "typeMap": {
+    "NUMERIC(38,18)": "String",
+    "NUMERIC(12,6)": "String",
+    "JSONB": "String",
+    "TIMESTAMPTZ": "DateTime",
+    "DATE": "DateTime"
+  },
+  "columns": {
+    "transactions.amount": "Transaction.amount",
+    "transactions.metadata": null,
+    "transactions.created_at": "Transaction.timestamp",
+    "transactions.updated_at": null,
+    "idempotency_keys.response_body": "IdempotencyKey.responseBody",
+    "idempotency_keys.locked_until": null,
+    "idempotency_keys.expires_at": "IdempotencyKey.expiresAt",
+    "idempotency_keys.created_at": "IdempotencyKey.createdAt",
+    "idempotency_keys.updated_at": null,
+    "vault_metrics_snapshots.total_assets": "SharePriceSnapshot.totalAssets",
+    "vault_metrics_snapshots.total_shares": "SharePriceSnapshot.totalShares",
+    "vault_metrics_snapshots.share_price": "SharePriceSnapshot.sharePrice",
+    "vault_metrics_snapshots.apy": null,
+    "vault_metrics_snapshots.recorded_at": "SharePriceSnapshot.recordedAt",
+    "apy_snapshots.date": null,
+    "apy_snapshots.apy": null,
+    "apy_snapshots.created_at": null
+  }
+}

backend/scripts/check-type-mapping.js

@@ -0,0 +1,106 @@
+'use strict';
+/**
+ * Static dev/prod type drift check (issue #1546).
+ *
+ * The dev database is Prisma on SQLite (`prisma/schema.prisma`), production
+ * storage is PostgreSQL (`migrations/*.sql`). SQLite has no NUMERIC, JSONB or
+ * TIMESTAMPTZ, so those columns are stored as Prisma `String` / `DateTime`.
+ * `prisma/postgres-type-mapping.json` records that mapping; this check fails
+ * when either side changes without the mapping being updated:
+ *
+ *   - a NUMERIC / JSONB / TIMESTAMPTZ / DATE column is not listed,
+ *   - a listed column or Prisma field no longer exists,
+ *   - a mapped Prisma field's type differs from the mapping for its Postgres type.
+ *
+ * Needs no database, so it runs in CI next to `db:check-drift`.
+ */
+const fs = require('fs');
+const path = require('path');
+
+const backendRoot = path.resolve(__dirname, '..');
+const DIVERGENT_TYPE = /^(NUMERIC|JSONB|TIMESTAMPTZ|DATE)\b/;
+
+/** Returns `{ "table.column": "PGTYPE" }` for every CREATE TABLE column. */
+function parsePostgresColumns(sql) {
+  const columns = {};
+  const tablePattern = /CREATE TABLE(?: IF NOT EXISTS)?\s+(\w+)\s*\(([\s\S]*?)\n\);/gi;
+  for (const [, table, body] of sql.matchAll(tablePattern)) {
+    for (const line of body.split('\n')) {
+      const match = line.trim().match(/^(\w+)\s+([A-Z]+(?:\s*\(\s*\d+\s*(?:,\s*\d+\s*)?\))?)/);
+      if (!match || /^(PRIMARY|UNIQUE|CHECK|CONSTRAINT|FOREIGN)$/i.test(match[1])) continue;
+      columns[`${table}.${match[1]}`] = match[2].replace(/\s+/g, '').toUpperCase();
+    }
+  }
+  return columns;
+}
+
+/** Returns `{ "Model.field": "Type" }` (optional `?` / list `[]` stripped). */
+function parsePrismaFields(schema) {
+  const fields = {};
+  for (const [, model, body] of schema.matchAll(/^model\s+(\w+)\s*\{([\s\S]*?)^\}/gm)) {
+    for (const line of body.split('\n')) {
+      const match = line.trim().match(/^(\w+)\s+(\w+)(\?|\[\])?/);
+      if (match) fields[`${model}.${match[1]}`] = match[2];
+    }
+  }
+  return fields;
+}
+
+function checkTypeMapping({ sql, schema, mapping }) {
+  const columns = parsePostgresColumns(sql);
+  const fields = parsePrismaFields(schema);
+  const errors = [];
+
+  for (const [column, pgType] of Object.entries(columns)) {
+    if (DIVERGENT_TYPE.test(pgType) && !(column in mapping.columns)) {
+      errors.push(`${column} (${pgType}) is not listed in postgres-type-mapping.json`);
+    }
+  }
+
+  for (const [column, field] of Object.entries(mapping.columns)) {
+    const pgType = columns[column];
+    if (!pgType) {
+      errors.push(`${column} is mapped but not defined in migrations/*.sql`);
+      continue;
+    }
+    if (field === null) continue;
+    const expected = mapping.typeMap[pgType];
+    if (!expected) {
+      errors.push(`${column}: no typeMap entry for ${pgType}`);
+    } else if (!fields[field]) {
+      errors.push(`${column}: Prisma field ${field} does not exist`);
+    } else if (fields[field] !== expected) {
+      errors.push(`${column} (${pgType}) maps to ${field} which is ${fields[field]}, expected ${expected}`);
+    }
+  }
+
+  return errors;
+}
+
+function loadSources(root = backendRoot) {
+  const migrationsDir = path.join(root, 'migrations');
+  const sql = fs
+    .readdirSync(migrationsDir)
+    .filter((file) => file.endsWith('.sql'))
+    .sort()
+    .map((file) => fs.readFileSync(path.join(migrationsDir, file), 'utf8'))
+    .join('\n');
+  return {
+    sql,
+    schema: fs.readFileSync(path.join(root, 'prisma', 'schema.prisma'), 'utf8'),
+    mapping: JSON.parse(
+      fs.readFileSync(path.join(root, 'prisma', 'postgres-type-mapping.json'), 'utf8'),
+    ),
+  };
+}
+
+if (require.main === module) {
+  const errors = checkTypeMapping(loadSources());
+  if (errors.length > 0) {
+    console.error(`Dev/prod type drift detected:\n- ${errors.join('\n- ')}`);
+    process.exit(1);
+  }
+  console.log('Prisma (SQLite) and PostgreSQL column types match postgres-type-mapping.json');
+}
+
+module.exports = { checkTypeMapping, loadSources, parsePostgresColumns, parsePrismaFields };

backend/src/__tests__/numericColumns.test.ts

@@ -0,0 +1,50 @@
+import Decimal from 'decimal.js';
+import { NumericDecimal, toNumeric38_18 } from '../numericColumns';
+import {
+  checkTypeMapping,
+  loadSources,
+} from '../../scripts/check-type-mapping';
+
+/** Largest value PostgreSQL NUMERIC(38, 18) can hold. */
+const MAX_NUMERIC = '99999999999999999999.999999999999999999';
+const SMALLEST_STEP = '0.000000000000000001';
+
+describe('NUMERIC(38, 18) precision (issue #1546)', () => {
+  it('round-trips the full 38-digit range through the String encoding', () => {
+    for (const value of [MAX_NUMERIC, `-${MAX_NUMERIC}`, SMALLEST_STEP, '0', '1.5']) {
+      expect(toNumeric38_18(value)).toBe(value);
+      expect(toNumeric38_18(toNumeric38_18(value))).toBe(value);
+    }
+  });
+
+  it('keeps sums exact where default Decimal precision rounds', () => {
+    const sum = new NumericDecimal('99999999999999999999.999999999999999998').plus(SMALLEST_STEP);
+    expect(toNumeric38_18(sum)).toBe(MAX_NUMERIC);
+
+    const lossy = new Decimal('99999999999999999999.999999999999999998').plus(SMALLEST_STEP);
+    expect(lossy.toFixed()).not.toBe(MAX_NUMERIC);
+  });
+
+  it('rejects values PostgreSQL would reject', () => {
+    expect(() => toNumeric38_18('100000000000000000000')).toThrow(RangeError);
+    expect(() => toNumeric38_18('0.0000000000000000001')).toThrow(RangeError);
+    expect(() => toNumeric38_18('Infinity')).toThrow(RangeError);
+    expect(() => toNumeric38_18('NaN')).toThrow(RangeError);
+  });
+});
+
+describe('dev/prod type mapping check', () => {
+  it('passes for the committed schema, migrations and mapping', () => {
+    expect(checkTypeMapping(loadSources())).toEqual([]);
+  });
+
+  it('flags an unlisted NUMERIC column and a mismatched Prisma type', () => {
+    const sources = loadSources();
+    const sql = `${sources.sql}\nCREATE TABLE IF NOT EXISTS fees (\n    total NUMERIC(38, 18) NOT NULL\n);`;
+    const schema = sources.schema.replace(/(model Transaction \{[\s\S]*?amount\s+)String/, '$1Float');
+    expect(checkTypeMapping({ ...sources, sql, schema })).toEqual([
+      'fees.total (NUMERIC(38,18)) is not listed in postgres-type-mapping.json',
+      'transactions.amount (NUMERIC(38,18)) maps to Transaction.amount which is Float, expected String',
+    ]);
+  });
+});