Run the integration suite in a throwaway schema and add CI

.github/workflows/ci.yml

@@ -0,0 +1,37 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      TEST_DB_URL: postgres://postgres:test@localhost:5432/postgres
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22.x
+          cache: npm
+      - run: npm ci
+      - run: npm run typecheck
+      - run: npm run typecheck:test
+      - run: npm test

package.json

@@ -13,7 +13,8 @@
     "start": "node dist/index.js",
     "test": "vitest run",
     "test:watch": "vitest",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit",
+    "typecheck:test": "tsc -p tsconfig.test.json"
   },
   "license": "Apache-2.0",
   "dependencies": {

test/api-recording.test.ts

@@ -140,7 +140,7 @@ describe("replay endpoints are never recorded", () => {
       recorder: captureRecorder,
       replay: {
         token: "t",
-        query: async () => ({ ts: [], kind: [], lat: [], lon: [], status: [], client: [], mapcode: [] }),
+        query: async () => ({ ts: [], kind: [], lat: [], lon: [], status: [], client: [], mapcode: [], country: [], anon: [] }),
         stats: async () => [],
         storage: async () => ({ databaseBytes: 0, tableBytes: 0, rowCount: 0 }),
       },

test/boundary-service.test.ts

@@ -21,9 +21,7 @@ import { BoundaryService, pointInRing } from "../src/domain/boundary-service.ts"
 // mocked module returns this value; otherwise it delegates to the real implementation.
 // This lets us exercise the load() CSR-build path with hand-built FeatureCollections
 // without disturbing the tests that read the real .fgb fixture.
-let deserializeOverride:
-  | ((bytes: Uint8Array) => unknown)
-  | null = null;
+let deserializeOverride: (() => unknown) | null = null;
 
 vi.mock("flatgeobuf", async (importOriginal) => {
   const actual = await importOriginal<typeof import("flatgeobuf")>();
@@ -33,7 +31,7 @@ vi.mock("flatgeobuf", async (importOriginal) => {
       ...actual.geojson,
       deserialize: (...args: Parameters<typeof actual.geojson.deserialize>) =>
         deserializeOverride
-          ? deserializeOverride(args[0] as Uint8Array)
+          ? deserializeOverride()
           : (actual.geojson.deserialize as (...a: unknown[]) => unknown)(...args),
     },
   };

test/db-integration.test.ts

@@ -17,25 +17,50 @@
 //   docker run --rm -d --name mapcode-test-pg -p 5433:5432 -e POSTGRES_PASSWORD=test postgres:16
 //   TEST_DB_URL=postgres://postgres:test@localhost:5433/postgres npx vitest run test/db-integration.test.ts
 //   docker stop mapcode-test-pg
-import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { randomBytes } from "node:crypto";
+import { describe, it, expect, beforeAll, beforeEach, afterAll } from "vitest";
 import { Pool } from "pg";
 import { ensureSchema, ensureBtreeIndex } from "../src/storage/schema.ts";
-import { createRequestRecorder } from "../src/storage/request-recorder.ts";
+import { createRequestRecorder, type RecordedRequest } from "../src/storage/request-recorder.ts";
 import { queryReplay } from "../src/storage/replay-query.ts";
 import { scanStats, querySizes } from "../src/storage/stats-query.ts";
 import { StatsCache } from "../src/storage/stats-cache.ts";
 
 const dbUrl = process.env.TEST_DB_URL;
+const schema = `mapcode_test_${randomBytes(6).toString("hex")}`;
+
+const GEO_ENCODE: RecordedRequest = { ts: 1000, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false };
+const DECODE: RecordedRequest = { ts: 1001, kind: 20, lat: 48858370, lon: 2294481, status: 20, client: 0, mapcode: "NLD 49.4V", country: null, anon: false };
+const NON_GEO: RecordedRequest = { ts: 1002, kind: 1, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false };
+const ANON_ENCODE: RecordedRequest = { ts: 1003, kind: 10, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: "NLD", anon: true };
+const ANON_NON_GEO: RecordedRequest = { ts: 1004, kind: 30, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: null, anon: true };
+const ANON_DECODE: RecordedRequest = { ts: 1005, kind: 20, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: "FRA", anon: true };
 
 describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => {
   let pool: Pool;
 
+  async function record(...rows: RecordedRequest[]): Promise<void> {
+    const recorder = createRequestRecorder(pool);
+    for (const row of rows) recorder.record(row);
+    await recorder.close();
+  }
+
   beforeAll(async () => {
-    pool = new Pool({ connectionString: dbUrl });
-    await pool.query("DROP TABLE IF EXISTS mapcode_request");
+    pool = new Pool({ connectionString: dbUrl, options: `-c search_path=${schema}` });
+    await pool.query(`CREATE SCHEMA ${schema}`);
+    const res = await pool.query("SELECT current_schema() AS name");
+    if (res.rows[0].name !== schema) {
+      throw new Error(`search_path not applied: current_schema() is ${res.rows[0].name}, expected ${schema}`);
+    }
+  });
+
+  beforeEach(async () => {
+    await pool.query(`DROP SCHEMA ${schema} CASCADE`);
+    await pool.query(`CREATE SCHEMA ${schema}`);
   });
 
   afterAll(async () => {
+    await pool?.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
     await pool?.end();
   });
 
@@ -47,11 +72,7 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => {
   });
 
   it("round-trips recorder inserts into the replay query", async () => {
-    const recorder = createRequestRecorder(pool);
-    recorder.record({ ts: 1000, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false });
-    recorder.record({ ts: 1001, kind: 20, lat: 48858370, lon: 2294481, status: 20, client: 0, mapcode: "NLD 49.4V", country: null, anon: false });
-    recorder.record({ ts: 1002, kind: 1, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }); // non-geo
-    await recorder.close();
+    await record(GEO_ENCODE, DECODE, NON_GEO);
 
     const cols = await queryReplay(pool, { from: 1000, to: 2000, limit: 10, kinds: null });
     expect(cols.ts).toEqual([1000, 1001]); // the full non-geo row is filtered out (lat NULL, not anon)
@@ -65,10 +86,7 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => {
   });
 
   it("stores anonymised rows with a country and anon, and returns them from replay without coords", async () => {
-    const recorder = createRequestRecorder(pool);
-    recorder.record({ ts: 1003, kind: 10, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: "NLD", anon: true });
-    recorder.record({ ts: 1004, kind: 30, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: null, anon: true });
-    await recorder.close();
+    await record(GEO_ENCODE, DECODE, NON_GEO, ANON_ENCODE, ANON_NON_GEO);
 
     const res = await pool.query("SELECT kind, client::int AS client, country, anon FROM mapcode_request WHERE ts = 1003");
     expect(res.rows).toEqual([{ kind: 10, client: 5, country: "NLD", anon: true }]);
@@ -82,32 +100,42 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => {
     expect(cols.anon).toEqual([0, 0, 1, 1]);
   });
 
-  it("backfills pre-marker anonymised rows on bootstrap, leaving full non-geo rows alone", async () => {
+  it("backfills pre-marker anonymised rows on bootstrap, leaving geo rows with a point and full non-geo rows alone", async () => {
     // Simulate a row written between the country column and the anon marker:
     // a geo kind without a point, anon still at its default.
+    await record(GEO_ENCODE, DECODE, NON_GEO);
     await pool.query("INSERT INTO mapcode_request (ts,lat,lon,kind,status,client,mapcode,country) VALUES (1005,NULL,NULL,20,20::int::\"char\",0::int::\"char\",NULL,'FRA')");
     await ensureSchema(pool);
-    const res = await pool.query("SELECT ts, anon FROM mapcode_request WHERE ts IN (1002, 1005) ORDER BY ts");
+    const res = await pool.query("SELECT ts, anon FROM mapcode_request ORDER BY ts");
     expect(res.rows).toEqual([
-      { ts: 1002, anon: false }, // full non-geo row (kind 1) is not a geo kind: untouched
+      { ts: GEO_ENCODE.ts, anon: false },
+      { ts: DECODE.ts, anon: false },
+      { ts: NON_GEO.ts, anon: false },
       { ts: 1005, anon: true },
     ]);
   });
 
   it("filters by kind, anonymised rows included", async () => {
+    await record(GEO_ENCODE, DECODE, NON_GEO, ANON_ENCODE, ANON_NON_GEO, ANON_DECODE);
     const cols = await queryReplay(pool, { from: 1000, to: 2000, limit: 10, kinds: [20] });
-    expect(cols.ts).toEqual([1001, 1005]); // the full decode and the backfilled anonymised one
+    expect(cols.ts).toEqual([DECODE.ts, ANON_DECODE.ts]);
     expect(cols.kind).toEqual([20, 20]);
     expect(cols.anon).toEqual([0, 1]);
   });
 
   it("rebuilds the stats cache from the tiered scan, non-geo rows included, replay rows in rowCount only", async () => {
     const now = Math.floor(Date.now() / 1000);
-    const recorder = createRequestRecorder(pool);
-    recorder.record({ ts: now, kind: 10, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false });
-    recorder.record({ ts: now - 7200, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false });
-    recorder.record({ ts: now, kind: 50, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }); // historical replay row
-    await recorder.close();
+    await record(
+      GEO_ENCODE,
+      DECODE,
+      NON_GEO,
+      ANON_ENCODE,
+      ANON_NON_GEO,
+      ANON_DECODE,
+      { ts: now, kind: 10, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false },
+      { ts: now - 7200, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false },
+      { ts: now, kind: 50, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false } // historical replay row
+    );
 
     const snap = StatsCache.fromScan(await scanStats(pool, now)).snapshot(now);
     const total = (key: "1m" | "1h" | "1d" | "all") => snap.rows.reduce((sum, r) => sum + r[key], 0);
@@ -117,20 +145,22 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => {
     expect(total("all")).toBe(8); // 3 replay round-trip rows + 3 anonymised rows + these 2; the kind-50 row is excluded
     expect(snap.rows.find((r) => r.kind === 10)?.all).toBe(4); // ts=1000 round-trip row + ts=1003 anonymised row + these 2
     expect(snap.rows.find((r) => r.kind === 10)?.anon.all).toBe(1); // only the ts=1003 row
-    expect(snap.rows.find((r) => r.kind === 20)?.anon.all).toBe(1); // the backfilled ts=1005 row
+    expect(snap.rows.find((r) => r.kind === 20)?.anon.all).toBe(1);
     expect(snap.rows.find((r) => r.kind === 30)?.anon.all).toBe(1); // the non-geo anonymised ts=1004 row
     expect(snap.rows.some((r) => r.kind === 50)).toBe(false);
     expect(snap.rowCount).toBe(9); // all rows, the kind-50 one included
   });
 
   it("measures storage sizes without a row count", async () => {
+    await record(GEO_ENCODE);
     const info = await querySizes(pool);
     expect(info.tableBytes).toBeGreaterThan(0);
     expect(info.databaseBytes).toBeGreaterThan(info.tableBytes);
     expect(info).not.toHaveProperty("rowCount");
   });
 
   it("uses the ts btree for the ordered range scan", async () => {
+    await ensureSchema(pool);
     await ensureBtreeIndex(pool);
     const client = await pool.connect();
     try {

test/server-logging.test.ts

@@ -239,7 +239,7 @@ describe("server logging", () => {
 
 const AUTH = { authorization: "Bearer test-token" };
 
-function failingReplay(): ServerDeps["replay"] {
+function failingReplay(): NonNullable<ServerDeps["replay"]> {
   return {
     token: "test-token",
     query: async () => {

tsconfig.test.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "rootDir": "."
+  },
+  "include": ["src", "test"]
+}