diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8a68f9c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DB_URL: postgres://postgres:test@localhost:5432/postgres + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22.x + cache: npm + - run: npm ci + - run: npm run typecheck + - run: npm run typecheck:test + - run: npm test diff --git a/package.json b/package.json index 72387b5..37d01d5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "start": "node dist/index.js", "test": "vitest run", "test:watch": "vitest", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "typecheck:test": "tsc -p tsconfig.test.json" }, "license": "Apache-2.0", "dependencies": { diff --git a/test/api-recording.test.ts b/test/api-recording.test.ts index b4cf708..55a0b92 100644 --- a/test/api-recording.test.ts +++ b/test/api-recording.test.ts @@ -140,7 +140,7 @@ describe("replay endpoints are never recorded", () => { recorder: captureRecorder, replay: { token: "t", - query: async () => ({ ts: [], kind: [], lat: [], lon: [], status: [], client: [], mapcode: [] }), + query: async () => ({ ts: [], kind: [], lat: [], lon: [], status: [], client: [], mapcode: [], country: [], anon: [] }), stats: async () => [], storage: async () => ({ databaseBytes: 0, tableBytes: 0, rowCount: 0 }), }, diff --git a/test/boundary-service.test.ts b/test/boundary-service.test.ts index 91f6c10..440d44f 100644 --- a/test/boundary-service.test.ts +++ b/test/boundary-service.test.ts @@ -21,9 +21,7 @@ import { BoundaryService, pointInRing } from "../src/domain/boundary-service.ts" // mocked module returns this value; otherwise it delegates to the real implementation. // This lets us exercise the load() CSR-build path with hand-built FeatureCollections // without disturbing the tests that read the real .fgb fixture. -let deserializeOverride: - | ((bytes: Uint8Array) => unknown) - | null = null; +let deserializeOverride: (() => unknown) | null = null; vi.mock("flatgeobuf", async (importOriginal) => { const actual = await importOriginal(); @@ -33,7 +31,7 @@ vi.mock("flatgeobuf", async (importOriginal) => { ...actual.geojson, deserialize: (...args: Parameters) => deserializeOverride - ? deserializeOverride(args[0] as Uint8Array) + ? deserializeOverride() : (actual.geojson.deserialize as (...a: unknown[]) => unknown)(...args), }, }; diff --git a/test/db-integration.test.ts b/test/db-integration.test.ts index 37d69c3..85d3521 100644 --- a/test/db-integration.test.ts +++ b/test/db-integration.test.ts @@ -17,25 +17,50 @@ // docker run --rm -d --name mapcode-test-pg -p 5433:5432 -e POSTGRES_PASSWORD=test postgres:16 // TEST_DB_URL=postgres://postgres:test@localhost:5433/postgres npx vitest run test/db-integration.test.ts // docker stop mapcode-test-pg -import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomBytes } from "node:crypto"; +import { describe, it, expect, beforeAll, beforeEach, afterAll } from "vitest"; import { Pool } from "pg"; import { ensureSchema, ensureBtreeIndex } from "../src/storage/schema.ts"; -import { createRequestRecorder } from "../src/storage/request-recorder.ts"; +import { createRequestRecorder, type RecordedRequest } from "../src/storage/request-recorder.ts"; import { queryReplay } from "../src/storage/replay-query.ts"; import { scanStats, querySizes } from "../src/storage/stats-query.ts"; import { StatsCache } from "../src/storage/stats-cache.ts"; const dbUrl = process.env.TEST_DB_URL; +const schema = `mapcode_test_${randomBytes(6).toString("hex")}`; + +const GEO_ENCODE: RecordedRequest = { ts: 1000, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false }; +const DECODE: RecordedRequest = { ts: 1001, kind: 20, lat: 48858370, lon: 2294481, status: 20, client: 0, mapcode: "NLD 49.4V", country: null, anon: false }; +const NON_GEO: RecordedRequest = { ts: 1002, kind: 1, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }; +const ANON_ENCODE: RecordedRequest = { ts: 1003, kind: 10, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: "NLD", anon: true }; +const ANON_NON_GEO: RecordedRequest = { ts: 1004, kind: 30, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: null, anon: true }; +const ANON_DECODE: RecordedRequest = { ts: 1005, kind: 20, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: "FRA", anon: true }; describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { let pool: Pool; + async function record(...rows: RecordedRequest[]): Promise { + const recorder = createRequestRecorder(pool); + for (const row of rows) recorder.record(row); + await recorder.close(); + } + beforeAll(async () => { - pool = new Pool({ connectionString: dbUrl }); - await pool.query("DROP TABLE IF EXISTS mapcode_request"); + pool = new Pool({ connectionString: dbUrl, options: `-c search_path=${schema}` }); + await pool.query(`CREATE SCHEMA ${schema}`); + const res = await pool.query("SELECT current_schema() AS name"); + if (res.rows[0].name !== schema) { + throw new Error(`search_path not applied: current_schema() is ${res.rows[0].name}, expected ${schema}`); + } + }); + + beforeEach(async () => { + await pool.query(`DROP SCHEMA ${schema} CASCADE`); + await pool.query(`CREATE SCHEMA ${schema}`); }); afterAll(async () => { + await pool?.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`); await pool?.end(); }); @@ -47,11 +72,7 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { }); it("round-trips recorder inserts into the replay query", async () => { - const recorder = createRequestRecorder(pool); - recorder.record({ ts: 1000, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false }); - recorder.record({ ts: 1001, kind: 20, lat: 48858370, lon: 2294481, status: 20, client: 0, mapcode: "NLD 49.4V", country: null, anon: false }); - recorder.record({ ts: 1002, kind: 1, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }); // non-geo - await recorder.close(); + await record(GEO_ENCODE, DECODE, NON_GEO); const cols = await queryReplay(pool, { from: 1000, to: 2000, limit: 10, kinds: null }); expect(cols.ts).toEqual([1000, 1001]); // the full non-geo row is filtered out (lat NULL, not anon) @@ -65,10 +86,7 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { }); it("stores anonymised rows with a country and anon, and returns them from replay without coords", async () => { - const recorder = createRequestRecorder(pool); - recorder.record({ ts: 1003, kind: 10, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: "NLD", anon: true }); - recorder.record({ ts: 1004, kind: 30, lat: null, lon: null, status: 20, client: 5, mapcode: null, country: null, anon: true }); - await recorder.close(); + await record(GEO_ENCODE, DECODE, NON_GEO, ANON_ENCODE, ANON_NON_GEO); const res = await pool.query("SELECT kind, client::int AS client, country, anon FROM mapcode_request WHERE ts = 1003"); expect(res.rows).toEqual([{ kind: 10, client: 5, country: "NLD", anon: true }]); @@ -82,32 +100,42 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { expect(cols.anon).toEqual([0, 0, 1, 1]); }); - it("backfills pre-marker anonymised rows on bootstrap, leaving full non-geo rows alone", async () => { + it("backfills pre-marker anonymised rows on bootstrap, leaving geo rows with a point and full non-geo rows alone", async () => { // Simulate a row written between the country column and the anon marker: // a geo kind without a point, anon still at its default. + await record(GEO_ENCODE, DECODE, NON_GEO); await pool.query("INSERT INTO mapcode_request (ts,lat,lon,kind,status,client,mapcode,country) VALUES (1005,NULL,NULL,20,20::int::\"char\",0::int::\"char\",NULL,'FRA')"); await ensureSchema(pool); - const res = await pool.query("SELECT ts, anon FROM mapcode_request WHERE ts IN (1002, 1005) ORDER BY ts"); + const res = await pool.query("SELECT ts, anon FROM mapcode_request ORDER BY ts"); expect(res.rows).toEqual([ - { ts: 1002, anon: false }, // full non-geo row (kind 1) is not a geo kind: untouched + { ts: GEO_ENCODE.ts, anon: false }, + { ts: DECODE.ts, anon: false }, + { ts: NON_GEO.ts, anon: false }, { ts: 1005, anon: true }, ]); }); it("filters by kind, anonymised rows included", async () => { + await record(GEO_ENCODE, DECODE, NON_GEO, ANON_ENCODE, ANON_NON_GEO, ANON_DECODE); const cols = await queryReplay(pool, { from: 1000, to: 2000, limit: 10, kinds: [20] }); - expect(cols.ts).toEqual([1001, 1005]); // the full decode and the backfilled anonymised one + expect(cols.ts).toEqual([DECODE.ts, ANON_DECODE.ts]); expect(cols.kind).toEqual([20, 20]); expect(cols.anon).toEqual([0, 1]); }); it("rebuilds the stats cache from the tiered scan, non-geo rows included, replay rows in rowCount only", async () => { const now = Math.floor(Date.now() / 1000); - const recorder = createRequestRecorder(pool); - recorder.record({ ts: now, kind: 10, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }); - recorder.record({ ts: now - 7200, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false }); - recorder.record({ ts: now, kind: 50, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }); // historical replay row - await recorder.close(); + await record( + GEO_ENCODE, + DECODE, + NON_GEO, + ANON_ENCODE, + ANON_NON_GEO, + ANON_DECODE, + { ts: now, kind: 10, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false }, + { ts: now - 7200, kind: 10, lat: 52376514, lon: 4908543, status: 20, client: 1, mapcode: null, country: null, anon: false }, + { ts: now, kind: 50, lat: null, lon: null, status: 20, client: 0, mapcode: null, country: null, anon: false } // historical replay row + ); const snap = StatsCache.fromScan(await scanStats(pool, now)).snapshot(now); const total = (key: "1m" | "1h" | "1d" | "all") => snap.rows.reduce((sum, r) => sum + r[key], 0); @@ -117,13 +145,14 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { expect(total("all")).toBe(8); // 3 replay round-trip rows + 3 anonymised rows + these 2; the kind-50 row is excluded expect(snap.rows.find((r) => r.kind === 10)?.all).toBe(4); // ts=1000 round-trip row + ts=1003 anonymised row + these 2 expect(snap.rows.find((r) => r.kind === 10)?.anon.all).toBe(1); // only the ts=1003 row - expect(snap.rows.find((r) => r.kind === 20)?.anon.all).toBe(1); // the backfilled ts=1005 row + expect(snap.rows.find((r) => r.kind === 20)?.anon.all).toBe(1); expect(snap.rows.find((r) => r.kind === 30)?.anon.all).toBe(1); // the non-geo anonymised ts=1004 row expect(snap.rows.some((r) => r.kind === 50)).toBe(false); expect(snap.rowCount).toBe(9); // all rows, the kind-50 one included }); it("measures storage sizes without a row count", async () => { + await record(GEO_ENCODE); const info = await querySizes(pool); expect(info.tableBytes).toBeGreaterThan(0); expect(info.databaseBytes).toBeGreaterThan(info.tableBytes); @@ -131,6 +160,7 @@ describe.skipIf(!dbUrl)("Postgres integration (TEST_DB_URL)", () => { }); it("uses the ts btree for the ordered range scan", async () => { + await ensureSchema(pool); await ensureBtreeIndex(pool); const client = await pool.connect(); try { diff --git a/test/server-logging.test.ts b/test/server-logging.test.ts index ad0ba51..57b5824 100644 --- a/test/server-logging.test.ts +++ b/test/server-logging.test.ts @@ -239,7 +239,7 @@ describe("server logging", () => { const AUTH = { authorization: "Bearer test-token" }; -function failingReplay(): ServerDeps["replay"] { +function failingReplay(): NonNullable { return { token: "test-token", query: async () => { diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..45bbe8f --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": "." + }, + "include": ["src", "test"] +}