From 94a2fb6a89ee860bf9a8d5134873b9267169ba2d Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:31:29 -0700 Subject: [PATCH 01/28] docs: record Container selection in ADR 0011 and the glossary. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 4 +++ docs/adr/0011-container-selection.md | 54 ++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 docs/adr/0011-container-selection.md diff --git a/CONTEXT.md b/CONTEXT.md index 0821b7e..99674c4 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -137,6 +137,10 @@ _Avoid_: Well (as the domain term); treating a missing position as occupying a s ### Export +**Container selection**: +The set of **Containers**, with their **Specimens**, that a set of filters matches: a **Study**, a subject list, or a micronix barcode list, narrowed by specimen type, collection date, container type, **Tags**, **Location**, and when the Specimen or the Container was created. Selecting several Tags matches Containers carrying every one of them. A barcode list selects exactly those tubes and takes no other filters. **Container export** and the statistics dashboard both count and list what the selection matches. +_Avoid_: Query (implementation term); treating Tag selection as "any of these Tags"; an unqualified "created" date (say Specimen created or Container created) + **Container export**: Server-side export of operational **Container** records with specimen, source, and placement context, formatted using an **Export configuration**. Entry points include **Multi-Study Export** (page heading **Export Containers (Multi-Study)**), the Study export modal, and **Micronix Barcode Export**. _Avoid_: Using "bulk export" as the umbrella for all container exports; collection table snapshot export; using "sample" in export column headers diff --git a/docs/adr/0011-container-selection.md b/docs/adr/0011-container-selection.md new file mode 100644 index 0000000..364f376 --- /dev/null +++ b/docs/adr/0011-container-selection.md @@ -0,0 +1,54 @@ +# Container selection: one module decides which Containers a filter matches + +--- +status: accepted +--- + +**Container export** and the statistics dashboard each turned their filters into a set of **Containers** their own way, and the filters drifted apart. Export matched Containers carrying every selected **Tag** while `GET /containers` matched any of them, and statistics re-implemented the intersection inline. "Created" meant `storageContainer.created` in export and `specimen.created` in statistics. The container-type filter ran up to twice per export, once in a `filterContainerIdsByType` call and again inside `enrichContainerData`, and the GET export returned "No containers found" before applying it. Multi-study export queried one Study at a time and invented a `MULTI` study record to satisfy enrichment. In statistics, the branch for location-only and tag-only filters could never run. + +We put **Container selection** (see [CONTEXT.md](../../CONTEXT.md)) behind one module, `packages/api/src/lib/container-selection/`. + +## Interface + +- One function takes a selection and returns the matching Container rows, the matching Specimen rows, and each Container's type. It never enriches: callers pass the rows to `loadContainerReadViews` ([ADR 0009](./0009-container-read-view.md)). +- A selection is one of: + - a **Study**; + - a subject list of (Study, Subject ids, per-subject dates) entries spanning any number of Studies; + - a micronix barcode list; + - the whole database. +- The study and subject-list selections are subject-only. The whole-database selection takes a source type filter (**Subject** or **Control batch**). A barcode list selects exactly those tubes and takes no other filters. +- Every other selection takes the same filters: + - specimen types (a list); + - collection date range; + - Specimen created; + - Container created; + - container types; + - Tags; + - **Location**. +- The module takes resolved ids. Callers resolve study codes, subject names and location ids, and return their own 404s. An unknown id inside a selection matches nothing. +- Queries stay bounded: subqueries or chunked IN lists, never an IN list that grows with the upload. + +## Rules the module owns + +- **Tags**: a Container matches when it carries every selected Tag. This holds for every caller, `GET /containers` included. +- **Created**: Specimen created and Container created are separate filters. The routes keep their wire names: export's `created_from`/`created_to` map to Container created, and statistics' to Specimen created. +- **Container type**: applied once, here. The GET export's "No containers found" 404 now fires on the filtered result, so a Study whose Containers are all of other types returns 404 instead of an empty file. +- **Location**: the Location and every Location under it. +- **Collection dates**: a listed Subject with its own dates ignores the global collection-date range. Undated Subjects get the global range. Tolerance widens exact dates only. +- **Specimens**: with no Container-level filter (container type, Tag, Location, Container created), every matching Specimen is included, with or without Containers. With one, only Specimens that have a matching Container are included. Export ignores Specimens with no Container. + +## Considered options + +- **"Any selected Tag" for `GET /containers`**: rejected. One meaning per filter is the point. No web caller sends `tag_ids` there, so only external clients are affected. +- **Container created everywhere**: rejected for now. It would silently change what the statistics dashboard counts. Naming both filters keeps current behaviour and makes the difference visible. +- **Unifying the Container export `run*` executors**: still a non-goal (ADR 0009). This module sits under them and only selects. + +## Consequences + +- `build*Query` in `lib/export/query.ts`, `filterContainerIdsByType`, `lib/container-tag-filter.ts`, and the tag intersection and dead branch in `dashboard-stats.ts` are deleted as their callers migrate. +- `enrichContainerData` loses its type filter and its `_specimens`, `study` and `subjectToStudyMap` parameters. It takes study fields from the Source's study chain ([ADR 0007](./0007-specimen-source-resolution.md)). The `MULTI` study record survives only in JSON export metadata. +- Callers build the **Export summary** from the module's output, grouping by Study for multi-study export. +- Migration is in slices, each deleting the code it replaces: the module with Container export, then dashboard statistics, then the `GET /containers` tag rule. +- The first slice adds a `CODING_STANDARDS.md` rule pointing new export and statistics code at this module, and lists the paths that haven't migrated yet. +- The statistics slice relabels the dashboard's "Created" filter as "Specimen created". The wire is unchanged. +- The module's tests cover each filter for every selection kind against the in-memory database. Statistics and export route tests keep aggregation and envelope checks. From cb80630c5dc2eaff1091446f084f68c1e1d64236 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:45:14 -0700 Subject: [PATCH 02/28] feat(api): add the Container selection module from ADR 0011. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/container-selection.test.ts | 562 ++++++++++++++++++ .../src/lib/container-selection/conditions.ts | 116 ++++ .../api/src/lib/container-selection/index.ts | 100 ++++ .../api/src/lib/container-selection/scopes.ts | 95 +++ .../api/src/lib/container-selection/types.ts | 72 +++ 5 files changed, 945 insertions(+) create mode 100644 packages/api/src/lib/container-selection/__tests__/container-selection.test.ts create mode 100644 packages/api/src/lib/container-selection/conditions.ts create mode 100644 packages/api/src/lib/container-selection/index.ts create mode 100644 packages/api/src/lib/container-selection/scopes.ts create mode 100644 packages/api/src/lib/container-selection/types.ts diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts new file mode 100644 index 0000000..05d3353 --- /dev/null +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -0,0 +1,562 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { count, eq } from 'drizzle-orm' +import type { Database as SQLiteDatabase } from 'bun:sqlite' +import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' +import { SQL_IN_CHUNK_SIZE } from '../../sql-helpers' +import { + createTestControlBatch, + createTestControlDefinition, + createTestCryovialBox, + createTestCryovialTube, + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestSpecimen, + createTestSpecimenType, + createTestStorageContainer, + createTestStorageType, + createTestStaticWell, + createTestPaper, + createTestSheet, + createTestBox, + createTestBag, + createTestStudy, + createTestStudySubject, + createTestTag, + createTestUnit, +} from '../../../__tests__/helpers/factories' +import type { Database } from '../../../db/client' +import { + specimen as specimenTable, + storageContainer as storageContainerTable, + storageContainerTag, +} from '../../../db/schema' +import { containerHasAllTags, selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '..' + +describe('selectContainers', () => { + let db: Database + let sqlite: SQLiteDatabase + let unitId: number + let specimenTypeId: number + let plateId: number + let cryovialBoxId: number + let storageTypeId: string + let barcodeCount = 0 + + beforeEach(async () => { + const setup = await setupTestDatabase() + db = setup.db + sqlite = setup.sqlite + unitId = (await createTestUnit(db, { symbol: 'uL', name: 'microliter', category: 'volume' })).id as number + specimenTypeId = (await createTestSpecimenType(db, { name: 'Blood' })).id + storageTypeId = String((await createTestStorageType(db, { name: 'Freezer -80' })).id) + const freezer = await createTestLocation(db, { name: 'Freezer', storageTypeId, canContainCollections: true }) + plateId = (await createTestMicronixPlate(db, { name: 'Plate', locationId: freezer.id })).id + cryovialBoxId = (await createTestCryovialBox(db, { name: 'Box', locationId: freezer.id })).id + }) + + afterEach(() => cleanupTestDatabase(sqlite)) + + async function seedSubject(studyId: number, name: string) { + return createTestStudySubject(db, { studyId, name }) + } + + async function seedSpecimen( + source: { studySubjectId: number } | { controlBatchId: number }, + fields: { collectionDate?: string; created?: string; specimenTypeId?: number } = {}, + ) { + const row = await createTestSpecimen(db, fields.specimenTypeId ?? specimenTypeId, source) + await db + .update(specimenTable) + .set({ + collectionDate: fields.collectionDate ?? null, + ...(fields.created ? { created: fields.created } : {}), + }) + .where(eq(specimenTable.id, row.id)) + return row + } + + type ContainerKind = 'micronix_tube' | 'cryovial_tube' | 'untyped' + + async function seedContainer( + specimenId: number, + options: { kind?: ContainerKind; created?: string; collectionId?: number; barcode?: string } = {}, + ) { + const container = await createTestStorageContainer(db, { specimenId, unitId }) + if (options.created) { + await db + .update(storageContainerTable) + .set({ created: options.created }) + .where(eq(storageContainerTable.id, container.id)) + } + const kind = options.kind ?? 'micronix_tube' + if (kind === 'micronix_tube') { + barcodeCount += 1 + await createTestMicronixTube(db, { + containerId: container.id, + collectionId: options.collectionId ?? plateId, + barcode: options.barcode ?? `MX${barcodeCount}`, + }) + } else if (kind === 'cryovial_tube') { + await createTestCryovialTube(db, { containerId: container.id, collectionId: options.collectionId ?? cryovialBoxId }) + } + return container + } + + async function tag(containerId: number, ...tagIds: number[]) { + await db.insert(storageContainerTag).values(tagIds.map((tagId) => ({ storageContainerId: containerId, tagId }))) + } + + const ids = (rows: Array<{ id: number }>) => rows.map((row) => row.id) + + describe('study selection', () => { + it("returns the Study's Containers, their Specimens and each Container's type", async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const other = await createTestStudy(db, { title: 'Other', shortCode: 'S2' }) + const subject = await seedSubject(study.id, 'P1') + const otherSubject = await seedSubject(other.id, 'Q1') + const specimen = await seedSpecimen({ studySubjectId: subject.id }) + const tube = await seedContainer(specimen.id, { kind: 'micronix_tube' }) + const vial = await seedContainer(specimen.id, { kind: 'cryovial_tube' }) + await seedContainer((await seedSpecimen({ studySubjectId: otherSubject.id })).id) + + const result = await selectContainers(db, { kind: 'study', studyId: study.id }) + + expect(ids(result.containers)).toEqual([tube.id, vial.id]) + expect(ids(result.specimens)).toEqual([specimen.id]) + expect(result.containerTypes).toEqual( + new Map([ + [tube.id, 'micronix_tube'], + [vial.id, 'cryovial_tube'], + ]), + ) + }) + + it('includes Specimens without Containers when no Container-level filter is set', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const stored = await seedSpecimen({ studySubjectId: subject.id }) + await seedContainer(stored.id) + const unstored = await seedSpecimen({ studySubjectId: subject.id }) + + const result = await selectContainers(db, { kind: 'study', studyId: study.id }) + + expect(ids(result.specimens)).toEqual([stored.id, unstored.id]) + expect(result.containers).toHaveLength(1) + }) + + it('matches nothing for an unknown Study id', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + await seedContainer((await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P1')).id })).id) + + const result = await selectContainers(db, { kind: 'study', studyId: study.id + 999 }) + + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + }) + }) + + const selectionKinds: Array< + [string, (studyId: number, subjectId: number, filters: ContainerSelectionFilters) => ContainerSelection] + > = [ + ['study', (studyId, _subjectId, filters) => ({ kind: 'study', studyId, filters })], + ['subject-list', (studyId, subjectId, filters) => ({ kind: 'subjects', subjects: [{ studyId, subjectId }], filters })], + ['whole-database', (_studyId, _subjectId, filters) => ({ kind: 'database', filters })], + ] + + describe.each(selectionKinds)('filters on a %s selection', (_kind, select) => { + it('keeps Specimens of the listed specimen types', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const plasmaTypeId = (await createTestSpecimenType(db, { name: 'Plasma' })).id + const dbsTypeId = (await createTestSpecimenType(db, { name: 'DBS' })).id + const blood = await seedSpecimen({ studySubjectId: subject.id }) + const plasma = await seedSpecimen({ studySubjectId: subject.id }, { specimenTypeId: plasmaTypeId }) + await seedSpecimen({ studySubjectId: subject.id }, { specimenTypeId: dbsTypeId }) + const bloodTube = await seedContainer(blood.id) + const plasmaTube = await seedContainer(plasma.id) + + const result = await selectContainers(db, select(study.id, subject.id, { specimenTypeIds: [specimenTypeId, plasmaTypeId] })) + + expect(ids(result.specimens)).toEqual([blood.id, plasma.id]) + expect(ids(result.containers)).toEqual([bloodTube.id, plasmaTube.id]) + }) + + it('keeps Specimens collected inside the collection date range, both ends inclusive', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-01-31' }) + const first = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-02-01' }) + const last = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-02-29' }) + await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-03-01' }) + await seedSpecimen({ studySubjectId: subject.id }) + + const result = await selectContainers(db, select(study.id, subject.id, { collectionDate: { from: '2024-02-01', to: '2024-02-29' } })) + + expect(ids(result.specimens)).toEqual([first.id, last.id]) + }) + + it('filters Specimen created on the Specimen, not on its Containers', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const january = await seedSpecimen({ studySubjectId: subject.id }, { created: '2024-01-15 10:00:00' }) + const januaryTube = await seedContainer(january.id, { created: '2024-06-01 09:00:00' }) + const june = await seedSpecimen({ studySubjectId: subject.id }, { created: '2024-06-01 09:00:00' }) + await seedContainer(june.id, { created: '2024-01-15 10:00:00' }) + + const result = await selectContainers(db, select(study.id, subject.id, { specimenCreated: { from: '2024-01-01', to: '2024-01-31' } })) + + expect(ids(result.specimens)).toEqual([january.id]) + expect(ids(result.containers)).toEqual([januaryTube.id]) + }) + + it('filters Container created on the Container, keeping only Specimens with a matching Container', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const january = await seedSpecimen({ studySubjectId: subject.id }, { created: '2024-06-01 09:00:00' }) + const januaryTube = await seedContainer(january.id, { created: '2024-01-31 23:59:59' }) + await seedContainer(january.id, { created: '2024-02-01 00:00:00' }) + const june = await seedSpecimen({ studySubjectId: subject.id }, { created: '2024-01-15 10:00:00' }) + await seedContainer(june.id, { created: '2024-06-01 09:00:00' }) + await seedSpecimen({ studySubjectId: subject.id }, { created: '2024-01-15 10:00:00' }) + + const result = await selectContainers(db, select(study.id, subject.id, { containerCreated: { from: '2024-01-01', to: '2024-01-31' } })) + + expect(ids(result.containers)).toEqual([januaryTube.id]) + expect(ids(result.specimens)).toEqual([january.id]) + }) + + it('keeps Containers of the listed container types, and only their Specimens', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const both = await seedSpecimen({ studySubjectId: subject.id }) + await seedContainer(both.id, { kind: 'micronix_tube' }) + const vial = await seedContainer(both.id, { kind: 'cryovial_tube' }) + const tubeOnly = await seedSpecimen({ studySubjectId: subject.id }) + await seedContainer(tubeOnly.id, { kind: 'micronix_tube' }) + const untypedOnly = await seedSpecimen({ studySubjectId: subject.id }) + const untyped = await seedContainer(untypedOnly.id, { kind: 'untyped' }) + + const vials = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['cryovial_tube', 'not_a_type'] })) + const unknown = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['unknown'] })) + + expect(ids(vials.containers)).toEqual([vial.id]) + expect(ids(vials.specimens)).toEqual([both.id]) + expect(vials.containerTypes).toEqual(new Map([[vial.id, 'cryovial_tube']])) + expect(ids(unknown.containers)).toEqual([untyped.id]) + expect(ids(unknown.specimens)).toEqual([untypedOnly.id]) + }) + + it('matches nothing when every listed container type is unknown', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + await seedContainer((await seedSpecimen({ studySubjectId: subject.id })).id) + + const result = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['not_a_type'] })) + + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + }) + + it('keeps Containers carrying every selected Tag, and only their Specimens', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const red = (await createTestTag(db, { name: 'red' })).id + const blue = (await createTestTag(db, { name: 'blue' })).id + const both = await seedSpecimen({ studySubjectId: subject.id }) + const bothTube = await seedContainer(both.id) + await tag(bothTube.id, red, blue) + const redOnly = await seedSpecimen({ studySubjectId: subject.id }) + await tag((await seedContainer(redOnly.id)).id, red) + + const result = await selectContainers(db, select(study.id, subject.id, { tagIds: [red, blue] })) + + expect(ids(result.containers)).toEqual([bothTube.id]) + expect(ids(result.specimens)).toEqual([both.id]) + }) + + it('matches nothing when a selected Tag id is unknown', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const red = (await createTestTag(db, { name: 'red' })).id + const subject = await seedSubject(study.id, 'P1') + const specimen = await seedSpecimen({ studySubjectId: subject.id }) + await tag((await seedContainer(specimen.id)).id, red) + + const result = await selectContainers(db, select(study.id, subject.id, { tagIds: [red, red + 999] })) + + expect(result.containers).toEqual([]) + }) + + it('keeps Containers stored in the Location or any Location under it, and only their Specimens', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const room = await createTestLocation(db, { name: 'Room', storageTypeId }) + const rack = await createTestLocation(db, { name: 'Rack', parentId: room.id, canContainCollections: true }) + const shelf = await createTestLocation(db, { name: 'Shelf', parentId: rack.id, canContainCollections: true }) + const shelfPlate = (await createTestMicronixPlate(db, { name: 'Shelf plate', locationId: shelf.id })).id + const rackBox = (await createTestCryovialBox(db, { name: 'Rack box', locationId: rack.id })).id + const shelfSheet = ( + await createTestSheet(db, { name: 'Sheet A', boxId: (await createTestBox(db, { name: 'Card box', locationId: shelf.id })).id }) + ).id + const rackSheet = ( + await createTestSheet(db, { name: 'Sheet B', bagId: (await createTestBag(db, { name: 'Bag', locationId: rack.id })).id }) + ).id + + const specimen = await seedSpecimen({ studySubjectId: subject.id }) + const tube = await seedContainer(specimen.id, { collectionId: shelfPlate }) + const vial = await seedContainer(specimen.id, { kind: 'cryovial_tube', collectionId: rackBox }) + const well = await seedContainer(specimen.id, { kind: 'untyped' }) + await createTestStaticWell(db, { containerId: well.id, collectionId: shelfPlate }) + const boxedPaper = await seedContainer(specimen.id, { kind: 'untyped' }) + await createTestPaper(db, { containerId: boxedPaper.id, sheetId: shelfSheet }) + const baggedPaper = await seedContainer(specimen.id, { kind: 'untyped' }) + await createTestPaper(db, { containerId: baggedPaper.id, sheetId: rackSheet }) + await seedContainer(specimen.id) + await seedContainer(specimen.id, { kind: 'untyped' }) + const elsewhere = await seedSpecimen({ studySubjectId: subject.id }) + await seedContainer(elsewhere.id) + + const underRack = await selectContainers(db, select(study.id, subject.id, { locationId: rack.id })) + const underShelf = await selectContainers(db, select(study.id, subject.id, { locationId: shelf.id })) + const unknown = await selectContainers(db, select(study.id, subject.id, { locationId: 99999 })) + + expect(ids(underRack.containers)).toEqual([tube.id, vial.id, well.id, boxedPaper.id, baggedPaper.id]) + expect(ids(underRack.specimens)).toEqual([specimen.id]) + expect(ids(underShelf.containers)).toEqual([tube.id, well.id, boxedPaper.id]) + expect(unknown).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + }) + }) + + describe('subject-list selection', () => { + it('selects the listed Subjects across Studies, and nothing for a Subject listed under the wrong Study or an unknown id', async () => { + const first = await createTestStudy(db, { title: 'First', shortCode: 'S1' }) + const second = await createTestStudy(db, { title: 'Second', shortCode: 'S2' }) + const p1 = await seedSubject(first.id, 'P1') + const p2 = await seedSubject(first.id, 'P2') + const q1 = await seedSubject(second.id, 'Q1') + const q2 = await seedSubject(second.id, 'Q2') + const p1Specimen = await seedSpecimen({ studySubjectId: p1.id }) + const p1Tube = await seedContainer(p1Specimen.id) + await seedContainer((await seedSpecimen({ studySubjectId: p2.id })).id) + const q1Specimen = await seedSpecimen({ studySubjectId: q1.id }) + const q1Tube = await seedContainer(q1Specimen.id) + await seedContainer((await seedSpecimen({ studySubjectId: q2.id })).id) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [ + { studyId: first.id, subjectId: p1.id }, + { studyId: second.id, subjectId: q1.id }, + { studyId: first.id, subjectId: q2.id }, + { studyId: first.id, subjectId: 99999 }, + ], + }) + + expect(ids(result.specimens)).toEqual([p1Specimen.id, q1Specimen.id]) + expect(ids(result.containers)).toEqual([p1Tube.id, q1Tube.id]) + }) + + it("uses a Subject's own dates instead of the global range, and the global range for undated Subjects", async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const dated = await seedSubject(study.id, 'P1') + const undated = await seedSubject(study.id, 'P2') + const visitOne = await seedSpecimen({ studySubjectId: dated.id }, { collectionDate: '2023-03-01' }) + const visitTwo = await seedSpecimen({ studySubjectId: dated.id }, { collectionDate: '2023-06-10' }) + await seedSpecimen({ studySubjectId: dated.id }, { collectionDate: '2024-01-15' }) + const inRange = await seedSpecimen({ studySubjectId: undated.id }, { collectionDate: '2024-01-15' }) + await seedSpecimen({ studySubjectId: undated.id }, { collectionDate: '2023-03-01' }) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [ + { studyId: study.id, subjectId: dated.id, dates: [{ exact: '2023-03-01' }, { from: '2023-06-01', to: '2023-06-30' }] }, + { studyId: study.id, subjectId: undated.id }, + ], + filters: { collectionDate: { from: '2024-01-01', to: '2024-12-31' } }, + }) + + expect(ids(result.specimens)).toEqual([visitOne.id, visitTwo.id, inRange.id]) + }) + + it('widens exact per-Subject dates by the tolerance, but not date ranges', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const exact = await seedSubject(study.id, 'P1') + const ranged = await seedSubject(study.id, 'P2') + const twoDaysBefore = await seedSpecimen({ studySubjectId: exact.id }, { collectionDate: '2024-02-28' }) + const twoDaysAfter = await seedSpecimen({ studySubjectId: exact.id }, { collectionDate: '2024-03-03' }) + await seedSpecimen({ studySubjectId: exact.id }, { collectionDate: '2024-03-04' }) + const rangeEnd = await seedSpecimen({ studySubjectId: ranged.id }, { collectionDate: '2024-03-31' }) + await seedSpecimen({ studySubjectId: ranged.id }, { collectionDate: '2024-04-01' }) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [ + { studyId: study.id, subjectId: exact.id, dates: [{ exact: '2024-03-01' }] }, + { studyId: study.id, subjectId: ranged.id, dates: [{ from: '2024-03-01', to: '2024-03-31' }] }, + ], + dateTolerance: 2, + }) + + expect(ids(result.specimens)).toEqual([twoDaysBefore.id, twoDaysAfter.id, rangeEnd.id]) + }) + + it('treats a Subject listed both with and without dates as undated', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const march = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-03-01' }) + const may = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-05-01' }) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [ + { studyId: study.id, subjectId: subject.id, dates: [{ exact: '2024-03-01' }] }, + { studyId: study.id, subjectId: subject.id }, + ], + }) + + expect(ids(result.specimens)).toEqual([march.id, may.id]) + }) + + it('selects nothing for an empty subject list', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + await seedContainer((await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P1')).id })).id) + + const result = await selectContainers(db, { kind: 'subjects', subjects: [] }) + + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + }) + + it('selects every listed Subject when the list is longer than one IN chunk', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const firstSubject = await seedSubject(study.id, 'P1') + const lastSubject = await seedSubject(study.id, 'P2') + const first = await seedSpecimen({ studySubjectId: firstSubject.id }, { collectionDate: '2024-01-01' }) + const last = await seedSpecimen({ studySubjectId: lastSubject.id }, { collectionDate: '2024-01-01' }) + const unknownSubjects = Array.from({ length: SQL_IN_CHUNK_SIZE }, (_, i) => ({ + studyId: study.id, + subjectId: 100000 + i, + dates: [{ exact: '2024-01-01' }], + })) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [ + { studyId: study.id, subjectId: firstSubject.id, dates: [{ exact: '2024-01-01' }] }, + ...unknownSubjects, + { studyId: study.id, subjectId: lastSubject.id }, + ], + }) + + expect(ids(result.specimens)).toEqual([first.id, last.id]) + }) + }) + + describe('whole-database selection', () => { + async function seedSources() { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subjectSpecimen = await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P1')).id }) + const definition = await createTestControlDefinition(db) + const batch = await createTestControlBatch(db, definition.id) + const controlSpecimen = await seedSpecimen({ controlBatchId: batch.id }) + return { subjectSpecimen, controlSpecimen } + } + + it('selects Specimens of Subjects and of Control batches', async () => { + const { subjectSpecimen, controlSpecimen } = await seedSources() + + const result = await selectContainers(db, { kind: 'database' }) + + expect(ids(result.specimens)).toEqual([subjectSpecimen.id, controlSpecimen.id]) + }) + + it('narrows to one source type', async () => { + const { subjectSpecimen, controlSpecimen } = await seedSources() + + const subjects = await selectContainers(db, { kind: 'database', sourceType: 'subject' }) + const controls = await selectContainers(db, { kind: 'database', sourceType: 'control' }) + + expect(ids(subjects.specimens)).toEqual([subjectSpecimen.id]) + expect(ids(controls.specimens)).toEqual([controlSpecimen.id]) + }) + }) + + describe('micronix barcode selection', () => { + it("selects exactly the listed tubes and their Specimens, whatever the Specimen's Source", async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subjectSpecimen = await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P1')).id }) + const listed = await seedContainer(subjectSpecimen.id, { barcode: 'A1' }) + await seedContainer(subjectSpecimen.id, { barcode: 'A2' }) + await seedContainer(subjectSpecimen.id, { kind: 'cryovial_tube' }) + const batch = await createTestControlBatch(db, (await createTestControlDefinition(db)).id) + const controlSpecimen = await seedSpecimen({ controlBatchId: batch.id }) + const control = await seedContainer(controlSpecimen.id, { barcode: 'C1' }) + await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P2')).id }) + + const result = await selectContainers(db, { kind: 'micronixBarcodes', barcodes: ['A1', 'C1', 'NOPE', 'A1'] }) + + expect(ids(result.containers)).toEqual([listed.id, control.id]) + expect(ids(result.specimens)).toEqual([subjectSpecimen.id, controlSpecimen.id]) + expect(result.containerTypes).toEqual( + new Map([ + [listed.id, 'micronix_tube'], + [control.id, 'micronix_tube'], + ]), + ) + }) + + it('selects every listed tube when the list is longer than one IN chunk', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const specimen = await seedSpecimen({ studySubjectId: (await seedSubject(study.id, 'P1')).id }) + const first = await seedContainer(specimen.id, { barcode: 'FIRST' }) + const last = await seedContainer(specimen.id, { barcode: 'LAST' }) + const unknown = Array.from({ length: SQL_IN_CHUNK_SIZE }, (_, i) => `UNKNOWN${i}`) + + const result = await selectContainers(db, { kind: 'micronixBarcodes', barcodes: ['FIRST', ...unknown, 'LAST'] }) + + expect(ids(result.containers)).toEqual([first.id, last.id]) + expect(ids(result.specimens)).toEqual([specimen.id]) + }) + }) +}) + +describe('containerHasAllTags', () => { + let db: Database + let sqlite: SQLiteDatabase + + beforeEach(async () => { + const setup = await setupTestDatabase() + db = setup.db + sqlite = setup.sqlite + }) + + afterEach(() => cleanupTestDatabase(sqlite)) + + it('filters and counts a paginated Container query by every selected Tag', async () => { + const red = (await createTestTag(db, { name: 'red' })).id + const blue = (await createTestTag(db, { name: 'blue' })).id + const unitId = (await createTestUnit(db, { symbol: 'uL', name: 'microliter', category: 'volume' })).id as number + const specimenTypeId = (await createTestSpecimenType(db, { name: 'Blood' })).id + const specimenId = (await createTestSpecimen(db, specimenTypeId)).id + const containers = [] + for (let i = 0; i < 4; i++) containers.push(await createTestStorageContainer(db, { specimenId, unitId })) + const links = [ + { storageContainerId: containers[0].id, tagId: red }, + { storageContainerId: containers[0].id, tagId: blue }, + { storageContainerId: containers[1].id, tagId: red }, + { storageContainerId: containers[2].id, tagId: red }, + { storageContainerId: containers[2].id, tagId: blue }, + ] + await db.insert(storageContainerTag).values(links) + + const where = containerHasAllTags([red, blue, red]) + const page = await db + .select() + .from(storageContainerTable) + .where(where) + .orderBy(storageContainerTable.id) + .limit(1) + .offset(1) + const [{ total }] = await db.select({ total: count() }).from(storageContainerTable).where(where) + + expect(page.map((row) => row.id)).toEqual([containers[2].id]) + expect(total).toBe(2) + }) +}) diff --git a/packages/api/src/lib/container-selection/conditions.ts b/packages/api/src/lib/container-selection/conditions.ts new file mode 100644 index 0000000..7af07de --- /dev/null +++ b/packages/api/src/lib/container-selection/conditions.ts @@ -0,0 +1,116 @@ +import { count, gte, inArray, sql, type SQL } from 'drizzle-orm' +import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' +import { + bag, + box, + cryovialBox, + cryovialTube, + location, + micronixPlate, + micronixTube, + paper, + sheet, + specimen, + staticWell, + storageContainer, + storageContainerTag, +} from '../../db/schema' +import type { ContainerSubtype } from '../container-placement' +import { dateToUpperBound } from '../sql-helpers' +import type { ContainerSelectionFilters, DateRange } from './types' + +/** + * A Container's type, from the subtype table holding it, in the precedence + * `resolveContainerPlacements` uses. 'unknown' when no subtype table holds it. + */ +export const containerTypeSql = sql`CASE + WHEN EXISTS (SELECT 1 FROM ${micronixTube} WHERE ${micronixTube.id} = ${storageContainer.id}) THEN 'micronix_tube' + WHEN EXISTS (SELECT 1 FROM ${cryovialTube} WHERE ${cryovialTube.id} = ${storageContainer.id}) THEN 'cryovial_tube' + WHEN EXISTS (SELECT 1 FROM ${paper} WHERE ${paper.id} = ${storageContainer.id}) THEN 'paper' + WHEN EXISTS (SELECT 1 FROM ${staticWell} WHERE ${staticWell.id} = ${storageContainer.id}) THEN 'static_well' + ELSE 'unknown' +END` + +/** Conditions for an inclusive date range on a date or timestamp column. */ +export function dateRangeConditions(column: SQLiteColumn, range: DateRange | undefined): SQL[] { + return [ + ...(range?.from ? [gte(column, range.from)] : []), + ...(range?.to ? [dateToUpperBound(column, range.to)] : []), + ] +} + +/** + * Conditions on the Specimen row other than its collection date, which depends on the + * selection: specimen types and Specimen created. + */ +export function specimenFilterConditions(filters: ContainerSelectionFilters): SQL[] { + const conditions: SQL[] = [] + if (filters.specimenTypeIds && filters.specimenTypeIds.length > 0) { + conditions.push(inArray(specimen.specimenTypeId, [...new Set(filters.specimenTypeIds)])) + } + conditions.push(...dateRangeConditions(specimen.created, filters.specimenCreated)) + return conditions +} + +/** + * Container-level filters: Container created, container types, Tags and Location. With any + * of them set, only Specimens with a matching Container are selected. + */ +export function containerFilterConditions(filters: ContainerSelectionFilters): SQL[] { + const conditions = [...dateRangeConditions(storageContainer.created, filters.containerCreated)] + if (filters.containerTypes && filters.containerTypes.length > 0) { + conditions.push(inArray(containerTypeSql, [...new Set(filters.containerTypes)])) + } + const tags = containerHasAllTags(filters.tagIds ?? []) + if (tags) conditions.push(tags) + if (filters.locationId !== undefined) conditions.push(containerAtLocation(filters.locationId)) + return conditions +} + +/** + * Containers stored in a Location or any Location under it: tubes and static wells on + * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags there. + */ +function containerAtLocation(locationId: number): SQL { + const here = sql`SELECT id FROM selected_location` + return sql`${storageContainer.id} IN ( + WITH RECURSIVE selected_location(id) AS ( + SELECT ${location.id} FROM ${location} WHERE ${location.id} = ${locationId} + UNION + SELECT child.id FROM ${location} AS child JOIN selected_location ON child.parent_id = selected_location.id + ) + SELECT ${micronixTube.id} FROM ${micronixTube} + JOIN ${micronixPlate} ON ${micronixPlate.id} = ${micronixTube.collectionId} + WHERE ${micronixPlate.locationId} IN (${here}) + UNION ALL + SELECT ${staticWell.id} FROM ${staticWell} + JOIN ${micronixPlate} ON ${micronixPlate.id} = ${staticWell.collectionId} + WHERE ${micronixPlate.locationId} IN (${here}) + UNION ALL + SELECT ${cryovialTube.id} FROM ${cryovialTube} + JOIN ${cryovialBox} ON ${cryovialBox.id} = ${cryovialTube.collectionId} + WHERE ${cryovialBox.locationId} IN (${here}) + UNION ALL + SELECT ${paper.id} FROM ${paper} + JOIN ${sheet} ON ${sheet.id} = ${paper.sheetId} + LEFT JOIN ${box} ON ${box.id} = ${sheet.boxId} + LEFT JOIN ${bag} ON ${bag.id} = ${sheet.bagId} + WHERE coalesce(${box.locationId}, ${bag.locationId}) IN (${here}) + )` +} + +/** + * The Tag rule (ADR 0011): a Container matches when it carries every selected Tag. A + * condition on `storage_container`, so a caller can page and count in SQL. Undefined when no + * Tag is selected; an unknown Tag id matches nothing. + */ +export function containerHasAllTags(tagIds: readonly number[]): SQL | undefined { + const unique = [...new Set(tagIds)] + if (unique.length === 0) return undefined + return sql`${storageContainer.id} IN ( + SELECT ${storageContainerTag.storageContainerId} FROM ${storageContainerTag} + WHERE ${inArray(storageContainerTag.tagId, unique)} + GROUP BY ${storageContainerTag.storageContainerId} + HAVING ${count()} = ${unique.length} + )` +} diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts new file mode 100644 index 0000000..4107540 --- /dev/null +++ b/packages/api/src/lib/container-selection/index.ts @@ -0,0 +1,100 @@ +/** + * Container selection (CONTEXT.md, ADR 0011): the Containers, with their Specimens, that a + * selection and its filters match. Every filter is defined once, here. The module selects + * only; callers enrich with `loadContainerReadViews` (ADR 0009). + */ +import { and, eq, getTableColumns, inArray, isNotNull, type SQL } from 'drizzle-orm' +import type { Database } from '../../db/client' +import { specimen, storageContainer, studySubject } from '../../db/schema' +import type { ContainerSubtype } from '../container-placement' +import { + containerFilterConditions, + containerTypeSql, + dateRangeConditions, + specimenFilterConditions, +} from './conditions' +import { micronixBarcodeScopes, subjectListScopes, type SelectionScope } from './scopes' +import type { ContainerSelection, ContainerSelectionResult, SelectedContainer, SelectedSpecimen } from './types' + +export * from './types' +export { containerHasAllTags } from './conditions' + +/** Select the Containers and Specimens a selection matches, with each Container's type. */ +export async function selectContainers( + database: Database, + selection: ContainerSelection, +): Promise { + if (selection.kind === 'micronixBarcodes') { + return runScopes(database, micronixBarcodeScopes(database, selection.barcodes), []) + } + + const filters = selection.filters ?? {} + const collectionDate = dateRangeConditions(specimen.collectionDate, filters.collectionDate) + let scopes: SelectionScope[] + switch (selection.kind) { + case 'study': { + const studySubjectIds = database + .select({ id: studySubject.id }) + .from(studySubject) + .where(eq(studySubject.studyId, selection.studyId)) + scopes = [{ specimens: and(inArray(specimen.studySubjectId, studySubjectIds), ...collectionDate) }] + break + } + case 'subjects': + scopes = await subjectListScopes(database, selection.subjects, { + collectionDate, + dateTolerance: selection.dateTolerance ?? 0, + }) + break + case 'database': { + const source = + selection.sourceType === 'subject' + ? isNotNull(specimen.studySubjectId) + : selection.sourceType === 'control' + ? isNotNull(specimen.controlBatchId) + : undefined + scopes = [{ specimens: and(source, ...collectionDate) }] + break + } + } + const specimenConditions = specimenFilterConditions(filters) + return runScopes( + database, + scopes.map((scope) => ({ ...scope, specimens: and(scope.specimens, ...specimenConditions) })), + containerFilterConditions(filters), + ) +} + +/** + * Run each scope (one per chunk of a listed selection) and collect its Specimens and the + * Containers that also pass the Container-level conditions. With any Container-level + * condition, only Specimens with a matching Container are kept. + */ +async function runScopes( + database: Database, + scopes: SelectionScope[], + containerConditions: SQL[], +): Promise { + const specimensById = new Map() + const containers: SelectedContainer[] = [] + const containerTypes = new Map() + for (const scope of scopes) { + for (const row of await database.select().from(specimen).where(scope.specimens)) specimensById.set(row.id, row) + const rows = await database + .select({ container: getTableColumns(storageContainer), containerType: containerTypeSql }) + .from(storageContainer) + .innerJoin(specimen, eq(storageContainer.specimenId, specimen.id)) + .where(and(scope.specimens, scope.containers, ...containerConditions)) + for (const row of rows) { + containers.push(row.container) + containerTypes.set(row.container.id, row.containerType) + } + } + let specimens = [...specimensById.values()] + if (containerConditions.length > 0) { + const stored = new Set(containers.map((container) => container.specimenId)) + specimens = specimens.filter((row) => stored.has(row.id)) + } + const byId = (a: { id: number }, b: { id: number }) => a.id - b.id + return { containers: containers.sort(byId), specimens: specimens.sort(byId), containerTypes } +} diff --git a/packages/api/src/lib/container-selection/scopes.ts b/packages/api/src/lib/container-selection/scopes.ts new file mode 100644 index 0000000..5e554c6 --- /dev/null +++ b/packages/api/src/lib/container-selection/scopes.ts @@ -0,0 +1,95 @@ +import { and, eq, inArray, or, type SQL } from 'drizzle-orm' +import type { SubjectDateFilter } from '@sampledb/contract' +import type { Database } from '../../db/client' +import { micronixTube, specimen, storageContainer, studySubject } from '../../db/schema' +import { shiftCalendarDate } from '../datetime' +import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' +import { dateRangeConditions } from './conditions' +import type { SubjectSelectionEntry } from './types' + +/** + * One bounded query's worth of a selection: the Specimens it covers and, for a barcode list, + * the Containers. A listed selection runs one scope per chunk, so no IN list grows with the + * input. + */ +export interface SelectionScope { + specimens: SQL | undefined + containers?: SQL +} + +/** Exactly the micronix tubes with these barcodes, and their Specimens, whatever their Source. */ +export function micronixBarcodeScopes(database: Database, barcodes: readonly string[]): SelectionScope[] { + const unique = [...new Set(barcodes.filter((barcode) => barcode !== ''))] + return chunkArray(unique, SQL_IN_CHUNK_SIZE).map((chunk) => { + const tubes = database.select({ id: micronixTube.id }).from(micronixTube).where(inArray(micronixTube.barcode, chunk)) + const tubeSpecimens = database + .select({ id: storageContainer.specimenId }) + .from(storageContainer) + .where(inArray(storageContainer.id, tubes)) + return { specimens: inArray(specimen.id, tubeSpecimens), containers: inArray(storageContainer.id, tubes) } + }) +} + +/** + * The listed Subjects, a chunk at a time. A Subject listed under a Study it isn't in matches + * nothing. A Subject with its own dates ignores the global collection-date range; one listed + * undated, on any entry, gets the global range. + */ +export async function subjectListScopes( + database: Database, + entries: readonly SubjectSelectionEntry[], + options: { collectionDate: SQL[]; dateTolerance: number }, +): Promise { + const datesBySubject = new Map() + for (const entry of entries) { + const listed = datesBySubject.get(entry.subjectId) ?? [] + let forStudy = listed.find((item) => item.studyId === entry.studyId) + if (!forStudy) { + forStudy = { studyId: entry.studyId, dates: [] } + listed.push(forStudy) + datesBySubject.set(entry.subjectId, listed) + } + const dateConditions = entry.dates ?? [] + if (dateConditions.length === 0) forStudy.dates = 'all' + else if (forStudy.dates !== 'all') forStudy.dates.push(...dateConditions) + } + + const scopes: SelectionScope[] = [] + for (const subjectIds of chunkArray([...datesBySubject.keys()], SQL_IN_CHUNK_SIZE)) { + const subjects = await database + .select({ id: studySubject.id, studyId: studySubject.studyId }) + .from(studySubject) + .where(inArray(studySubject.id, subjectIds)) + const undated: number[] = [] + const dated: SQL[] = [] + for (const subject of subjects) { + const listed = datesBySubject.get(subject.id)?.find((item) => item.studyId === subject.studyId) + if (!listed) continue + const dateConditions = + listed.dates === 'all' + ? [] + : listed.dates + .map((dateFilter) => subjectDateCondition(dateFilter, options.dateTolerance)) + .filter((condition) => condition !== undefined) + if (dateConditions.length === 0) undated.push(subject.id) + else dated.push(and(eq(specimen.studySubjectId, subject.id), or(...dateConditions))!) + } + if (undated.length > 0) { + dated.push(and(inArray(specimen.studySubjectId, undated), ...options.collectionDate)!) + } + if (dated.length > 0) scopes.push({ specimens: or(...dated) }) + } + return scopes +} + +/** Collection date condition for one per-Subject date: an exact date widened by the tolerance, or a range. */ +function subjectDateCondition(dateFilter: SubjectDateFilter, toleranceDays: number): SQL | undefined { + if ('exact' in dateFilter) { + const from = shiftCalendarDate(dateFilter.exact, -toleranceDays) + const to = shiftCalendarDate(dateFilter.exact, toleranceDays) + if (!from || !to) return eq(specimen.collectionDate, dateFilter.exact) + return and(...dateRangeConditions(specimen.collectionDate, { from, to })) + } + const range = dateRangeConditions(specimen.collectionDate, dateFilter) + return range.length > 0 ? and(...range) : undefined +} diff --git a/packages/api/src/lib/container-selection/types.ts b/packages/api/src/lib/container-selection/types.ts new file mode 100644 index 0000000..398be34 --- /dev/null +++ b/packages/api/src/lib/container-selection/types.ts @@ -0,0 +1,72 @@ +import type { SubjectDateFilter } from '@sampledb/contract' +import type { specimen, storageContainer } from '../../db/schema' +import type { ContainerSubtype } from '../container-placement' + +/** An inclusive date range. A bare `YYYY-MM-DD` upper bound includes that whole day. */ +export interface DateRange { + from?: string + to?: string +} + +/** Filters every selection takes, except a micronix barcode list (ADR 0011). */ +export interface ContainerSelectionFilters { + /** Specimen types; empty or omitted means every type. */ + specimenTypeIds?: readonly number[] + /** Collection date range. A listed Subject with its own dates ignores it. */ + collectionDate?: DateRange + /** When the Specimen was created. */ + specimenCreated?: DateRange + /** When the Container was created. A Container-level filter. */ + containerCreated?: DateRange + /** Container types (`ContainerSubtype` values); unknown values match nothing. A Container-level filter. */ + containerTypes?: readonly string[] + /** Containers carrying every one of these Tags. A Container-level filter. */ + tagIds?: readonly number[] + /** Containers stored in this Location or any Location under it. A Container-level filter. */ + locationId?: number +} + +/** One listed Subject of a subject-list selection. */ +export interface SubjectSelectionEntry { + studyId: number + subjectId: number + /** + * This Subject's collection dates; several are alternatives (one per visit). Omitted or + * empty: the global collection-date range applies. When a Subject is listed more than + * once, an undated entry wins. + */ + dates?: readonly SubjectDateFilter[] +} + +/** What to select from (ADR 0011). Every id is resolved by the caller; an unknown id matches nothing. */ +export type ContainerSelection = + | { kind: 'study'; studyId: number; filters?: ContainerSelectionFilters } + | { + kind: 'subjects' + subjects: readonly SubjectSelectionEntry[] + /** Days either side of each exact per-Subject date. Ranges are not widened. */ + dateTolerance?: number + filters?: ContainerSelectionFilters + } + | { kind: 'micronixBarcodes'; barcodes: readonly string[] } + | { + kind: 'database' + /** Specimens of Subjects or of Control batches; omitted means both. */ + sourceType?: 'subject' | 'control' + filters?: ContainerSelectionFilters + } + +export type SelectedContainer = typeof storageContainer.$inferSelect +export type SelectedSpecimen = typeof specimen.$inferSelect + +export interface ContainerSelectionResult { + /** Matching Containers, ordered by id. */ + containers: SelectedContainer[] + /** + * Matching Specimens, ordered by id. With no Container-level filter, every matching + * Specimen, with or without Containers; with one, only Specimens that have a matching Container. + */ + specimens: SelectedSpecimen[] + /** Each selected Container's type, by Container id. */ + containerTypes: Map +} From b5513702badaf501985007d9974dd1eab6f84906 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:50:10 -0700 Subject: [PATCH 03/28] feat(api): match every selected Tag in GET /containers. Breaking: tag_ids now lists only Containers carrying all the selected Tags, using containerHasAllTags from Container selection (ADR 0011). Pagination and the total stay in SQL. Co-Authored-By: Claude Opus 5.5 --- .../src/routes/__tests__/containers.test.ts | 71 ++++++++++++++++++- packages/api/src/routes/containers.ts | 30 ++------ 2 files changed, 75 insertions(+), 26 deletions(-) diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index ead182c..c56dd06 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -18,8 +18,10 @@ import { createTestStorageType, createTestCryovialBox, createTestCryovialTube, + createTestStorageContainer, + createTestTag, } from '../../__tests__/helpers/factories' -import { cryovialBox, cryovialTube, specimen, storageContainer, micronixPlate, micronixTube, box, sheet, paper, staticWell } from '../../db/schema' +import { cryovialBox, cryovialTube, specimen, storageContainer, micronixPlate, micronixTube, box, sheet, paper, staticWell, storageContainerTag } from '../../db/schema' import { eq } from 'drizzle-orm' describe('Containers API', () => { @@ -71,6 +73,73 @@ describe('Containers API', () => { expect(data.details).toBe('enrichment failed') vi.restoreAllMocks() }) + + describe('tag_ids filter', () => { + type ListResponse = { + containers: Array<{ id: number }> + pagination: { page: number; limit: number; total: number; totalPages: number } + } + + async function containerTagged(unitId: number, specimenId: number, tagIds: number[]) { + const container = await createTestStorageContainer(ctx.db, { unitId, specimenId }) + if (tagIds.length > 0) { + await ctx.db + .insert(storageContainerTag) + .values(tagIds.map((tagId) => ({ storageContainerId: container.id, tagId }))) + } + return container.id + } + + async function list(query: string): Promise { + const res = await ctx.request(`/api/containers?${query}`) + expect(res.status).toBe(200) + return (await res.json()) as ListResponse + } + + let red: number + let blue: number + let unused: number + let redAndBlue: number[] + let redOnly: number + + beforeEach(async () => { + red = (await createTestTag(ctx.db, { name: 'red' })).id + blue = (await createTestTag(ctx.db, { name: 'blue' })).id + unused = (await createTestTag(ctx.db, { name: 'unused' })).id + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const testSpecimen = await createTestSpecimen(ctx.db, specimenType.id) + redAndBlue = [ + await containerTagged(unit.id, testSpecimen.id, [red, blue]), + await containerTagged(unit.id, testSpecimen.id, [red, blue]), + await containerTagged(unit.id, testSpecimen.id, [red, blue]), + ] + redOnly = await containerTagged(unit.id, testSpecimen.id, [red]) + await containerTagged(unit.id, testSpecimen.id, [blue]) + await containerTagged(unit.id, testSpecimen.id, []) + }) + + it('lists Containers carrying the one selected Tag', async () => { + const data = await list(`tag_ids=${red}&limit=10`) + expect(data.containers.map((c) => c.id).sort((a, b) => a - b)).toEqual([...redAndBlue, redOnly]) + expect(data.pagination.total).toBe(4) + }) + + it('excludes Containers with only some of the selected Tags from the page and the total', async () => { + const first = await list(`tag_ids=${red}&tag_ids=${blue}&limit=2&page=1`) + const second = await list(`tag_ids=${red}&tag_ids=${blue}&limit=2&page=2`) + expect([...first.containers, ...second.containers].map((c) => c.id)).toEqual(redAndBlue) + expect(first.containers).toHaveLength(2) + expect(first.pagination).toEqual({ page: 1, limit: 2, total: 3, totalPages: 2 }) + expect(second.pagination).toEqual({ page: 2, limit: 2, total: 3, totalPages: 2 }) + }) + + it('returns an empty page when no Container carries every selected Tag', async () => { + const data = await list(`tag_ids=${red}&tag_ids=${unused}&limit=2`) + expect(data.containers).toEqual([]) + expect(data.pagination).toEqual({ page: 1, limit: 2, total: 0, totalPages: 0 }) + }) + }) }) describe('GET /api/containers/:id', () => { diff --git a/packages/api/src/routes/containers.ts b/packages/api/src/routes/containers.ts index f0bb191..71539db 100644 --- a/packages/api/src/routes/containers.ts +++ b/packages/api/src/routes/containers.ts @@ -1,7 +1,7 @@ import { Hono } from 'hono' import type { Database } from '../db/client' import { storageContainer, micronixTube, cryovialTube, paper, staticWell, tag, storageContainerTag } from '../db/schema' -import { eq, and, sql, inArray } from 'drizzle-orm' +import { eq, and, sql } from 'drizzle-orm' import { z } from 'zod' import { validatePage, validateLimit } from '../lib/constants' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' @@ -12,6 +12,7 @@ import { enrichContainerForApi, enrichContainersForApi, toEnrichedContainerApi } import { loadContainerReadViews } from '../lib/container-read-view' import { mapEnrichedContainerToWire, mapEnrichedContainersToWire } from '../lib/container-wire-mapper' import { handleRouteError, RouteError, containersFetchFailedBody } from '../lib/error-handler' +import { containerHasAllTags } from '../lib/container-selection' /** * Create containers routes with database injection @@ -44,30 +45,9 @@ containers.get('/', authMiddleware, async (c) => { } } - // Filter by tags if provided - if (tagIds && tagIds.length > 0) { - const containerIdsWithTags = await database - .select({ containerId: storageContainerTag.storageContainerId }) - .from(storageContainerTag) - .where(inArray(storageContainerTag.tagId, tagIds)) - - const ids = [...new Set(containerIdsWithTags.map(r => r.containerId))] - if (ids.length > 0) { - conditions.push(inArray(storageContainer.id, ids)) - } else { - // No containers match the tags, return empty result - return c.json({ - containers: [], - pagination: { - page, - limit, - total: 0, - totalPages: 0, - }, - }) - } - } - + const tags = containerHasAllTags(tagIds ?? []) + if (tags) conditions.push(tags) + if (conditions.length > 0) { const whereClause = and(...conditions) as any query = query.where(whereClause) as any From de3dc5605937e10a96602600bb38e1d63392f64c Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:50:24 -0700 Subject: [PATCH 04/28] feat(export): select single-study GET export through Container selection. `GET /export/containers` now picks its Containers with `selectContainers` (ADR 0011), so the container type filter runs once, in the selection. A Study whose Containers are all of other types now returns 404 "No containers found for this study" instead of an empty file. `created_from` and `created_to` map to Container created. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/export/container-export.ts | 61 ++++++++++------ .../api/src/routes/__tests__/export.test.ts | 71 +++++++++++++++++++ 2 files changed, 109 insertions(+), 23 deletions(-) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index 05d62d3..6992cea 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,5 +1,8 @@ import type { ContainerExportData } from '@sampledb/contract' +import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' +import { study } from '../../db/schema' +import { selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '../container-selection' import { ExpectedNotFoundError, RouteError } from '../error-handler' import { enrichContainerData } from './enrich' import { filterContainerIdsByType } from './filter' @@ -83,6 +86,24 @@ export function buildPostExportEnvelope(params: { } } +/** + * Container selection filters for an export request (ADR 0011). Export's + * `created_from`/`created_to` mean Container created. + */ +function exportSelectionFilters(filters: Omit): ContainerSelectionFilters { + return { + specimenTypeIds: filters.specimen_type_ids, + collectionDate: { from: filters.date_from, to: filters.date_to }, + containerCreated: { from: filters.created_from, to: filters.created_to }, + containerTypes: filters.container_types, + tagIds: filters.tag_ids, + } +} + +async function findStudyByShortCode(database: Database, shortCode: string): Promise { + return database.select().from(study).where(eq(study.shortCode, shortCode)).get() +} + async function formatContainerExport( database: Database, data: ContainerExportData[], @@ -124,22 +145,23 @@ export async function runSingleStudyContainerExportGet( | { kind: 'json'; json: Awaited>; filename: string } > { const { filters, format, columns, csvOptions, userId, countOnly } = params - let result - try { - result = await buildContainerQuery(database, filters) - } catch (error: unknown) { - const message = error instanceof Error ? error.message : 'Not found' - throw new ExpectedNotFoundError(message) + const studyRecord = await findStudyByShortCode(database, filters.study) + if (!studyRecord) { + throw new ExpectedNotFoundError(`Study with short code '${filters.study}' not found`) } + const selectionFilters = exportSelectionFilters(filters) + const selection: ContainerSelection = filters.subject_ids?.length + ? { + kind: 'subjects', + subjects: filters.subject_ids.map((subjectId) => ({ studyId: studyRecord.id, subjectId })), + filters: selectionFilters, + } + : { kind: 'study', studyId: studyRecord.id, filters: selectionFilters } + const result = await selectContainers(database, selection) + if (countOnly) { - let filteredContainers = result.containers - if (filters.container_types?.length && filteredContainers.length > 0) { - const containerIds = filteredContainers.map((c) => c.id) - const matchingIds = await filterContainerIdsByType(database, containerIds, filters.container_types) - filteredContainers = filteredContainers.filter((c) => matchingIds.includes(c.id)) - } - return { kind: 'count', count: filteredContainers.length } + return { kind: 'count', count: result.containers.length } } if (result.containers.length === 0) { @@ -151,20 +173,13 @@ export async function runSingleStudyContainerExportGet( throw new RouteError(400, { error: 'Invalid format. Use csv, xlsx, or json.' }) } - const enrichedData = await enrichContainerData( - database, - result.containers, - result.specimens ?? [], - result.study, - filters.container_types, - undefined - ) + const enrichedData = await enrichContainerData(database, result.containers, result.specimens, studyRecord) const timestamp = formatExportTimestamp() - const filenameBase = `study_${result.study.shortCode}_export_${timestamp}` + const filenameBase = `study_${studyRecord.shortCode}_export_${timestamp}` if (format === 'json') { - const json = await formatAsJSON(database, enrichedData, filters, result.study, columns, userId) + const json = await formatAsJSON(database, enrichedData, filters, studyRecord, columns, userId) return { kind: 'json', json, filename: `${filenameBase}.json` } } diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 31ca23e..9516c94 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -14,6 +14,13 @@ import { createTestSpecimenType, createTestSpecimen, createTestStorageContainer, + createTestUnit, + createTestStorageType, + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestCryovialBox, + createTestCryovialTube, } from '../../__tests__/helpers/factories' describe('Export API', () => { @@ -37,6 +44,32 @@ describe('Export API', () => { ctx.cleanup() }) + /** One Subject in a new Study with a micronix tube and, optionally, a cryovial tube. */ + async function seedTypedContainers(shortCode: string, options: { cryovial: boolean }) { + const study = await createTestStudy(ctx.db, { title: `${shortCode} Study`, shortCode }) + const subject = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Subj1' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer -80' }) + const freezer = await createTestLocation(ctx.db, { + name: 'Freezer', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'Plate', locationId: freezer.id }) + const spec = await createTestSpecimen(ctx.db, specimenType.id, { studySubjectId: subject.id }) + const micronix = await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id as number }) + await createTestMicronixTube(ctx.db, { containerId: micronix.id, collectionId: plate.id, barcode: `${shortCode}-MX` }) + let cryovialId: number | undefined + if (options.cryovial) { + const box = await createTestCryovialBox(ctx.db, { name: 'Box', locationId: freezer.id }) + const cryovial = await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id as number }) + await createTestCryovialTube(ctx.db, { containerId: cryovial.id, collectionId: box.id }) + cryovialId = cryovial.id + } + return { micronixId: micronix.id, cryovialId } + } + describe('GET /api/export/specimens.csv', () => { it('returns 200 with auth and CSV content type', async () => { const res = await ctx.request('/api/export/specimens.csv', { @@ -193,6 +226,44 @@ describe('Export API', () => { expect(rowsAfter.length).toBe(rowsBefore.length) }) + it('exports only Containers of the requested container_types', async () => { + const { cryovialId } = await seedTypedContainers('TYPES', { cryovial: true }) + + const res = await ctx.request('/api/export/containers?study=TYPES&format=json&container_types=cryovial_tube', { + method: 'GET', + }) + expect(res.status).toBe(200) + const data = (await res.json()) as { + export_metadata: { count: number } + containers: Array<{ container_id: number; container_type: string }> + } + expect(data.containers.map((row) => [row.container_id, row.container_type])).toEqual([ + [cryovialId, 'cryovial_tube'], + ]) + expect(data.export_metadata.count).toBe(1) + + const countRes = await ctx.request('/api/export/containers?study=TYPES&count_only=true&container_types=cryovial_tube', { + method: 'GET', + }) + expect(await countRes.json()).toEqual({ count: 1 }) + }) + + it('returns 404 when every Container in the study is of another container type', async () => { + await seedTypedContainers('MXONLY', { cryovial: false }) + + const res = await ctx.request('/api/export/containers?study=MXONLY&format=csv&container_types=cryovial_tube', { + method: 'GET', + }) + expect(res.status).toBe(404) + const data = (await res.json()) as { error?: string } + expect(data.error).toBe('No containers found for this study') + + const countRes = await ctx.request('/api/export/containers?study=MXONLY&count_only=true&container_types=cryovial_tube', { + method: 'GET', + }) + expect(await countRes.json()).toEqual({ count: 0 }) + }) + it('returns 400 for invalid format', async () => { const study = await createTestStudy(ctx.db, { title: 'Fmt Study', shortCode: 'FMT' }) const subject = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Subj1' }) From 4ce22ee8ee45c62a94f9d2e2400764ff4be04c5f Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:50:55 -0700 Subject: [PATCH 05/28] feat(web): label the dashboard's created filter Specimen created. The statistics created_from/created_to filter has always matched the Specimen's created date. The filter fields and their chips now say so, in the wording ADR 0011 uses. Co-Authored-By: Claude Opus 5.5 --- .../web/src/components/StatisticsFilter.tsx | 8 ++++---- packages/web/src/pages/Statistics.tsx | 4 ++-- .../web/src/pages/__tests__/Statistics.test.tsx | 17 +++++++++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/packages/web/src/components/StatisticsFilter.tsx b/packages/web/src/components/StatisticsFilter.tsx index aab5dd0..202539c 100644 --- a/packages/web/src/components/StatisticsFilter.tsx +++ b/packages/web/src/components/StatisticsFilter.tsx @@ -480,10 +480,10 @@ export default function StatisticsFilter({ filters, onChange, onSubmit, isLoadin /> - {/* Created From */} + {/* Specimen Created From */}
- {/* Created To */} + {/* Specimen Created To */}
0) { const name = f.locationSelections?.[0]?.name ?? `Location ${f.locationId ?? ''}` chips.push({ key: 'locationId', label: `Location: ${name}` }) diff --git a/packages/web/src/pages/__tests__/Statistics.test.tsx b/packages/web/src/pages/__tests__/Statistics.test.tsx index c2c8970..d9fd233 100644 --- a/packages/web/src/pages/__tests__/Statistics.test.tsx +++ b/packages/web/src/pages/__tests__/Statistics.test.tsx @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' +import userEvent from '@testing-library/user-event' import { render, screen, waitFor } from '../../__tests__/helpers/render' import Statistics from '../Statistics' import { statisticsApi } from '../../lib/api/statistics' @@ -71,6 +72,22 @@ describe('Statistics', () => { }) }) + it('filters and labels the date range as Specimen created', async () => { + const user = userEvent.setup() + await render() + await user.click(await screen.findByRole('button', { name: /filters/i })) + + await user.type(screen.getByLabelText('Specimen Created From'), '2024-01-01') + await user.type(screen.getByLabelText('Specimen Created To'), '2024-12-31') + await user.click(screen.getByRole('button', { name: /apply filters/i })) + + expect(await screen.findByText('Specimen created from: 2024-01-01')).toBeInTheDocument() + expect(screen.getByText('Specimen created to: 2024-12-31')).toBeInTheDocument() + expect(statisticsApi.get).toHaveBeenLastCalledWith( + expect.objectContaining({ created_from: '2024-01-01', created_to: '2024-12-31' }), + ) + }) + // We do not defend against malformed API responses: the server is under our control // and must return the full StatisticsData shape; malformed data indicates a server bug and we fail loudly. }) From b53074dcf1c884c34123ed132ef24c8ef5801885 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:51:35 -0700 Subject: [PATCH 06/28] fix(export): return the rows from JSON POST container exports. formatContainerExport returned the JSON rows as `json`, but buildPostExportEnvelope reads `jsonData`, so every POST container export in JSON format sent an empty `data` array. New executor tests for multi-study and barcode export pin the rows, the per-Study summary and the barcode summary before those paths move to Container selection. Co-Authored-By: Claude Opus 5.5 --- .../export/__tests__/container-export.test.ts | 300 ++++++++++++++++++ .../api/src/lib/export/container-export.ts | 4 +- 2 files changed, 302 insertions(+), 2 deletions(-) create mode 100644 packages/api/src/lib/export/__tests__/container-export.test.ts diff --git a/packages/api/src/lib/export/__tests__/container-export.test.ts b/packages/api/src/lib/export/__tests__/container-export.test.ts new file mode 100644 index 0000000..5c2a169 --- /dev/null +++ b/packages/api/src/lib/export/__tests__/container-export.test.ts @@ -0,0 +1,300 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { eq } from 'drizzle-orm' +import type { Database as SQLiteDatabase } from 'bun:sqlite' +import type { ContainerExportData } from '@sampledb/contract' +import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' +import { + createTestControlBatch, + createTestControlDefinition, + createTestCryovialBox, + createTestCryovialTube, + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestSpecimen, + createTestSpecimenType, + createTestStorageContainer, + createTestStorageType, + createTestStudy, + createTestStudySubject, + createTestTag, + createTestUnit, +} from '../../../__tests__/helpers/factories' +import type { Database } from '../../../db/client' +import { specimen as specimenTable, storageContainerTag } from '../../../db/schema' +import { runBarcodeContainerExportPost, runMultiStudyContainerExportPost } from '../container-export' +import type { MultiStudyExportSummary } from '../types' + +const columns = ['container_id', 'container_type', 'barcode', 'subject_name', 'study_code', 'control_batch_name'] + +describe('Container export executors', () => { + let db: Database + let sqlite: SQLiteDatabase + let unitId: number + let specimenTypeId: number + let plateId: number + let cryovialBoxId: number + let barcodeCount = 0 + + beforeEach(async () => { + const setup = await setupTestDatabase() + db = setup.db + sqlite = setup.sqlite + unitId = (await createTestUnit(db, { symbol: 'uL', name: 'microliter', category: 'volume' })).id as number + specimenTypeId = (await createTestSpecimenType(db, { name: 'Blood' })).id + const storageTypeId = String((await createTestStorageType(db, { name: 'Freezer -80' })).id) + const freezer = await createTestLocation(db, { name: 'Freezer', storageTypeId, canContainCollections: true }) + plateId = (await createTestMicronixPlate(db, { name: 'Plate', locationId: freezer.id })).id + cryovialBoxId = (await createTestCryovialBox(db, { name: 'Box', locationId: freezer.id })).id + }) + + afterEach(() => cleanupTestDatabase(sqlite)) + + async function seedSpecimen( + source: { studySubjectId: number } | { controlBatchId: number }, + collectionDate?: string, + ) { + const row = await createTestSpecimen(db, specimenTypeId, source) + await db + .update(specimenTable) + .set({ collectionDate: collectionDate ?? null }) + .where(eq(specimenTable.id, row.id)) + return row + } + + async function seedTube(specimenId: number, barcode?: string) { + const container = await createTestStorageContainer(db, { specimenId, unitId }) + barcodeCount += 1 + await createTestMicronixTube(db, { + containerId: container.id, + collectionId: plateId, + barcode: barcode ?? `MX${barcodeCount}`, + }) + return container + } + + async function seedCryovial(specimenId: number) { + const container = await createTestStorageContainer(db, { specimenId, unitId }) + await createTestCryovialTube(db, { containerId: container.id, collectionId: cryovialBoxId }) + return container + } + + const containerIds = (data: unknown) => + (data as ContainerExportData[]).map((row) => row.container_id).sort((a, b) => a - b) + + describe('runMultiStudyContainerExportPost', () => { + /** Two Studies sharing a Subject name, with visits, a cryovial and an unknown Subject. */ + async function seedTwoStudies() { + const alpha = await createTestStudy(db, { title: 'Alpha', shortCode: 'ALPHA', leadPerson: 'Ada' }) + const beta = await createTestStudy(db, { title: 'Beta', shortCode: 'BETA', leadPerson: 'Bo' }) + const alphaS01 = await createTestStudySubject(db, { studyId: alpha.id, name: 'S01' }) + const alphaS02 = await createTestStudySubject(db, { studyId: alpha.id, name: 'S02' }) + const betaS01 = await createTestStudySubject(db, { studyId: beta.id, name: 'S01' }) + const alphaJan = await seedTube((await seedSpecimen({ studySubjectId: alphaS01.id }, '2024-01-01')).id) + await seedTube((await seedSpecimen({ studySubjectId: alphaS01.id }, '2024-03-01')).id) + const alphaS02Cryovial = await seedCryovial((await seedSpecimen({ studySubjectId: alphaS02.id }, '2024-01-01')).id) + const betaSpecimen = await seedSpecimen({ studySubjectId: betaS01.id }, '2024-06-01') + const betaTube = await seedTube(betaSpecimen.id) + const betaCryovial = await seedCryovial(betaSpecimen.id) + return { alphaJan, alphaS02Cryovial, betaTube, betaCryovial } + } + + const entries = [ + { study_short_code: 'ALPHA', subject_name: 'S01', collection_date: '2024-01-01' }, + { study_short_code: 'ALPHA', subject_name: 'S02' }, + { study_short_code: 'ALPHA', subject_name: 'NOBODY' }, + { study_short_code: 'BETA', subject_name: 'S01' }, + ] + + it('exports every listed Subject across Studies and summarises each Study', async () => { + const { alphaJan, alphaS02Cryovial, betaTube, betaCryovial } = await seedTwoStudies() + + const result = await runMultiStudyContainerExportPost(db, { entries, filters: {}, format: 'json', columns }) + + if (!('data' in result)) throw new Error('expected an export envelope') + expect(containerIds(result.data)).toEqual([alphaJan.id, alphaS02Cryovial.id, betaTube.id, betaCryovial.id]) + const studyCodeByContainer = new Map( + (result.data as ContainerExportData[]).map((row) => [row.container_id, row.study_code]), + ) + expect(studyCodeByContainer.get(alphaJan.id)).toBe('ALPHA') + expect(studyCodeByContainer.get(betaTube.id)).toBe('BETA') + expect(result.summary).toEqual({ + total_containers: 4, + studies: [ + { + study_code: 'ALPHA', + study_title: 'Alpha', + study_lead_person: 'Ada', + containers: 2, + subjects_with_results: [ + { name: 'S01', count: 1 }, + { name: 'S02', count: 1 }, + ], + subjects_no_results: [], + subjects_not_found: ['NOBODY'], + }, + { + study_code: 'BETA', + study_title: 'Beta', + study_lead_person: 'Bo', + containers: 2, + subjects_with_results: [{ name: 'S01', count: 2 }], + subjects_no_results: [], + subjects_not_found: [], + }, + ], + invalid_study_codes: [], + } satisfies MultiStudyExportSummary) + }) + + it('applies the container type filter once and counts Subjects left without Containers', async () => { + const { alphaJan, betaTube } = await seedTwoStudies() + + const result = await runMultiStudyContainerExportPost(db, { + entries, + filters: { container_types: ['micronix_tube'] }, + format: 'json', + columns, + }) + + if (!('data' in result)) throw new Error('expected an export envelope') + expect(containerIds(result.data)).toEqual([alphaJan.id, betaTube.id]) + const summary = result.summary as MultiStudyExportSummary + expect(summary.total_containers).toBe(2) + expect(summary.studies[0]).toMatchObject({ + containers: 1, + subjects_with_results: [{ name: 'S01', count: 1 }], + subjects_no_results: ['S02'], + subjects_not_found: ['NOBODY'], + }) + expect(summary.studies[1]).toMatchObject({ containers: 1, subjects_with_results: [{ name: 'S01', count: 1 }] }) + }) + + it('keeps only Containers carrying every selected Tag', async () => { + const { alphaJan, betaTube } = await seedTwoStudies() + const first = await createTestTag(db, { name: 'first' }) + const second = await createTestTag(db, { name: 'second' }) + await db.insert(storageContainerTag).values([ + { storageContainerId: alphaJan.id, tagId: first.id }, + { storageContainerId: alphaJan.id, tagId: second.id }, + { storageContainerId: betaTube.id, tagId: first.id }, + ]) + + const result = await runMultiStudyContainerExportPost(db, { + entries, + filters: { tag_ids: [first.id, second.id] }, + format: 'json', + columns, + }) + + if (!('data' in result)) throw new Error('expected an export envelope') + expect(containerIds(result.data)).toEqual([alphaJan.id]) + }) + + it('counts with the same summary as the export', async () => { + await seedTwoStudies() + + const exported = await runMultiStudyContainerExportPost(db, { entries, filters: {}, format: 'json', columns }) + const counted = await runMultiStudyContainerExportPost(db, { + entries, + filters: {}, + format: 'json', + countOnly: true, + }) + + expect(counted).toEqual({ count: 4, summary: exported.summary as MultiStudyExportSummary }) + }) + + it('reports a Study whose Subjects are all unknown', async () => { + await seedTwoStudies() + + const result = await runMultiStudyContainerExportPost(db, { + entries: [{ study_short_code: 'BETA', subject_name: 'NOBODY' }], + filters: {}, + format: 'json', + countOnly: true, + }) + + expect(result).toEqual({ + count: 0, + summary: { + total_containers: 0, + studies: [ + { + study_code: 'BETA', + study_title: 'Beta', + study_lead_person: 'Bo', + containers: 0, + subjects_with_results: [], + subjects_no_results: [], + subjects_not_found: ['NOBODY'], + }, + ], + invalid_study_codes: [], + }, + }) + }) + + it('exports nothing when a study code is invalid', async () => { + await seedTwoStudies() + + const result = await runMultiStudyContainerExportPost(db, { + entries: [...entries, { study_short_code: 'NOPE', subject_name: 'S01' }], + filters: {}, + format: 'json', + countOnly: true, + }) + + expect(result).toEqual({ + count: 0, + summary: { + total_containers: 0, + studies: [], + invalid_study_codes: ['NOPE'], + errors: ['Invalid study codes: NOPE'], + }, + }) + }) + }) + + describe('runBarcodeContainerExportPost', () => { + it('exports exactly the listed tubes, whatever their Source, and reports unknown barcodes', async () => { + const alpha = await createTestStudy(db, { title: 'Alpha', shortCode: 'ALPHA' }) + const beta = await createTestStudy(db, { title: 'Beta', shortCode: 'BETA' }) + const alphaSubject = await createTestStudySubject(db, { studyId: alpha.id, name: 'S01' }) + const betaSubject = await createTestStudySubject(db, { studyId: beta.id, name: 'S01' }) + const batch = await createTestControlBatch(db, (await createTestControlDefinition(db)).id, { name: 'Batch 1' }) + const alphaSpecimen = await seedSpecimen({ studySubjectId: alphaSubject.id }) + const alphaTube = await seedTube(alphaSpecimen.id, 'A1') + await seedTube(alphaSpecimen.id, 'A2') + const betaTube = await seedTube((await seedSpecimen({ studySubjectId: betaSubject.id })).id, 'B1') + const controlTube = await seedTube((await seedSpecimen({ controlBatchId: batch.id })).id, 'C1') + + const result = await runBarcodeContainerExportPost(db, { + barcodes: ['A1', 'B1', 'C1', 'NOPE', 'A1'], + format: 'json', + columns, + }) + + expect(result.summary).toEqual({ + total_containers: 3, + barcodes_found: ['A1', 'B1', 'C1'], + barcodes_not_found: ['NOPE'], + }) + const rows = result.data as ContainerExportData[] + expect(containerIds(rows)).toEqual([alphaTube.id, betaTube.id, controlTube.id]) + const byId = new Map(rows.map((row) => [row.container_id, row])) + expect(byId.get(alphaTube.id)?.study_code).toBe('ALPHA') + expect(byId.get(betaTube.id)?.study_code).toBe('BETA') + expect(byId.get(controlTube.id)).toMatchObject({ control_batch_name: 'Batch 1' }) + expect(byId.get(controlTube.id)?.study_code).toBeUndefined() + }) + + it('returns an empty export when no barcode matches', async () => { + const result = await runBarcodeContainerExportPost(db, { barcodes: ['NOPE'], format: 'json', columns }) + + expect(result.summary).toEqual({ total_containers: 0, barcodes_found: [], barcodes_not_found: ['NOPE'] }) + expect(result.data).toEqual([]) + }) + }) +}) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index 05d62d3..ede5bc5 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -92,10 +92,10 @@ async function formatContainerExport( columns: string[] | undefined, csvOptions: CSVExportOptions | undefined, userId: number | null | undefined -): Promise<{ csv?: string; xlsx?: Buffer; json?: ContainerExportData[] }> { +): Promise<{ csv?: string; xlsx?: Buffer; jsonData?: ContainerExportData[] }> { if (format === 'json') { const json = await formatAsJSON(database, data, filters, study, columns, userId) - return { json: json.containers } + return { jsonData: json.containers } } if (format === 'csv') { From 1bf007648de4f3af00c5da4b53094c45900ff6ca Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:53:04 -0700 Subject: [PATCH 07/28] feat(export): select single-study POST export through Container selection. `POST /export/containers` now builds a subject-list selection from the resolved subject names and their per-subject dates, and selects with `selectContainers` (ADR 0011). Count-only and full export share the one selection, so neither calls `filterContainerIdsByType` any more. The summary is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/export/container-export.ts | 83 ++++++++----------- .../api/src/routes/__tests__/export.test.ts | 78 ++++++++++++++++- 2 files changed, 110 insertions(+), 51 deletions(-) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index 6992cea..0cdd08f 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,15 +1,19 @@ -import type { ContainerExportData } from '@sampledb/contract' +import type { ContainerExportData, SubjectDateFilter } from '@sampledb/contract' import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' import { study } from '../../db/schema' -import { selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '../container-selection' +import { + selectContainers, + type ContainerSelection, + type ContainerSelectionFilters, + type SubjectSelectionEntry, +} from '../container-selection' import { ExpectedNotFoundError, RouteError } from '../error-handler' import { enrichContainerData } from './enrich' -import { filterContainerIdsByType } from './filter' import { formatAsCSV, formatAsExcel, formatAsJSON } from './format' -import { buildContainerQuery, buildContainerQueryByMicronixBarcodes, buildMultiStudyContainerQuery, resolveMicronixBarcodesToContainers } from './query' +import { buildContainerQueryByMicronixBarcodes, buildMultiStudyContainerQuery, resolveMicronixBarcodesToContainers } from './query' import { buildExportSummary, validateStudyCodes } from './validate' -import { resolveSubjectNamesByStudy, resolveStudyByShortCode } from '../identifier-resolution' +import { resolveSubjectNamesByStudy } from '../identifier-resolution' import type { CSVExportOptions, ExportFilters, @@ -203,7 +207,8 @@ export async function runSingleStudyContainerExportPost( csvOptions?: CSVExportOptions userId?: number | null countOnly?: boolean - subjectDates?: ExportFilters['subject_dates'] + /** Per-Subject collection dates, keyed by subject name. */ + subjectDates?: Record dateTolerance?: number } ): Promise { @@ -220,19 +225,18 @@ export async function runSingleStudyContainerExportPost( dateTolerance = 0, } = params - const studyId = await resolveStudyByShortCode(database, studyCode) - if (!studyId) { + const studyRecord = await findStudyByShortCode(database, studyCode) + if (!studyRecord) { throw new ExpectedNotFoundError('Study not found') } - const subjectNameToId = await resolveSubjectNamesByStudy(database, subjectNames, studyId) - const subjectIds = Array.from(subjectNameToId.values()) + const subjectNameToId = await resolveSubjectNamesByStudy(database, subjectNames, studyRecord.id) const subjectIdToName = new Map() for (const [name, id] of subjectNameToId.entries()) { subjectIdToName.set(id, name) } - if (subjectIds.length === 0) { + if (subjectNameToId.size === 0) { const summary: ExportSummary = { total_containers: 0, subjects_with_results: [], @@ -255,55 +259,34 @@ export async function runSingleStudyContainerExportPost( }) } - const queryFilters: ExportFilters = { - ...filters, - study: studyCode, - subject_ids: subjectIds, - date_tolerance: dateTolerance, - } - - if (subjectDates) { - queryFilters.subject_dates = {} - for (const [subjectName, dateFilter] of Object.entries(subjectDates)) { - const subjectId = subjectNameToId.get(subjectName) - if (subjectId) { - queryFilters.subject_dates![subjectId] = dateFilter as NonNullable[number] - } + const subjects: SubjectSelectionEntry[] = [...subjectNameToId].map(([name, subjectId]) => { + const dates = subjectDates?.[name] + return { + studyId: studyRecord.id, + subjectId, + dates: dates === undefined ? undefined : Array.isArray(dates) ? dates : [dates], } - } - - const result = await buildContainerQuery(database, queryFilters).catch((error: unknown) => { - const message = error instanceof Error ? error.message : 'Not found' - throw new ExpectedNotFoundError(message) + }) + const result = await selectContainers(database, { + kind: 'subjects', + subjects, + dateTolerance, + filters: exportSelectionFilters(filters), }) if (countOnly) { - let filteredContainers = result.containers - if (queryFilters.container_types?.length) { - const containerIds = result.containers.map((c) => c.id) - const matchingIds = await filterContainerIdsByType(database, containerIds, queryFilters.container_types) - filteredContainers = result.containers.filter((c) => matchingIds.includes(c.id)) - } - const summary: ExportSummary = { - total_containers: filteredContainers.length, + total_containers: result.containers.length, subjects_with_results: [], subjects_no_results: [], subjects_not_found: subjectNames.filter((name) => !subjectNameToId.has(name)), errors: [], } - return { count: filteredContainers.length, summary } + return { count: result.containers.length, summary } } - const enrichedData = await enrichContainerData( - database, - result.containers, - result.specimens ?? [], - result.study, - queryFilters.container_types, - undefined - ) + const enrichedData = await enrichContainerData(database, result.containers, result.specimens, studyRecord) const summary = await buildExportSummary( enrichedData, @@ -316,8 +299,8 @@ export async function runSingleStudyContainerExportPost( database, enrichedData, format, - queryFilters, - result.study, + { ...filters, study: studyCode, date_tolerance: dateTolerance }, + studyRecord, columns, csvOptions, userId @@ -325,7 +308,7 @@ export async function runSingleStudyContainerExportPost( return buildPostExportEnvelope({ format, - filenameBase: `study_${result.study.shortCode}_export_${formatExportTimestamp()}`, + filenameBase: `study_${studyRecord.shortCode}_export_${formatExportTimestamp()}`, summary, ...formatted, }) diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 9516c94..d94ed31 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -6,7 +6,7 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createExportRoutes } from '../export' -import { errorLogs } from '../../db/schema' +import { errorLogs, specimen as specimenTable } from '../../db/schema' import * as specimensCsv from '../../lib/export/specimens-csv' import { createTestStudy, @@ -314,6 +314,82 @@ describe('Export API', () => { }) }) + describe('POST /api/export/containers filters', () => { + type PostSummary = { + total_containers: number + subjects_with_results: Array<{ name: string; count: number }> + subjects_no_results: string[] + subjects_not_found: string[] + } + + /** POST a CSV export; return its summary and each row's container_id and container_type. */ + async function postCsvExport(body: Record) { + const res = await ctx.request('/api/export/containers', { + method: 'POST', + json: { ...body, format: 'csv', csv_bom: false, csv_line_ending: 'LF' }, + }) + expect(res.status).toBe(200) + const data = (await res.json()) as { summary: PostSummary; data: string } + const [header, ...lines] = Buffer.from(data.data, 'base64').toString('utf8').trim().split('\n') + expect(header.startsWith('container_id,container_type,')).toBe(true) + const rows = lines.map((line) => { + const [id, type] = line.split(',') + return [Number(id), type] as const + }) + return { summary: data.summary, rows } + } + + it('exports and counts only Containers of the requested container_types', async () => { + const { cryovialId } = await seedTypedContainers('PTYPES', { cryovial: true }) + const body = { study: 'PTYPES', subject_names: ['Subj1', 'Ghost'], container_types: ['cryovial_tube'] } + + const exported = await postCsvExport(body) + expect(exported.rows).toEqual([[cryovialId, 'cryovial_tube']]) + expect(exported.summary).toEqual({ + total_containers: 1, + subjects_with_results: [{ name: 'Subj1', count: 1 }], + subjects_no_results: [], + subjects_not_found: ['Ghost'], + }) + + const countRes = await ctx.request('/api/export/containers', { + method: 'POST', + json: { ...body, count_only: true }, + }) + const counted = (await countRes.json()) as { count: number; summary: PostSummary } + expect(counted.count).toBe(1) + expect(counted.summary.total_containers).toBe(1) + expect(counted.summary.subjects_not_found).toEqual(['Ghost']) + }) + + it('applies per-subject dates with tolerance, and the global range to undated subjects', async () => { + const study = await createTestStudy(ctx.db, { title: 'Dates Study', shortCode: 'PDATES' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const containerOn = async (subjectId: number, collectionDate: string) => { + const spec = await createTestSpecimen(ctx.db, specimenType.id, { studySubjectId: subjectId }) + await ctx.db.update(specimenTable).set({ collectionDate }).where(eq(specimenTable.id, spec.id)) + return (await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id as number })).id + } + const dated = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Dated' }) + const undated = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Undated' }) + const datedJan = await containerOn(dated.id, '2024-01-01') + await containerOn(dated.id, '2024-03-01') + await containerOn(undated.id, '2024-01-15') + const undatedFeb = await containerOn(undated.id, '2024-02-15') + + const exported = await postCsvExport({ + study: 'PDATES', + subject_names: ['Dated', 'Undated'], + subject_dates: { Dated: { exact: '2024-01-02' } }, + date_tolerance: 1, + date_from: '2024-02-01', + }) + expect(exported.rows.map(([id]) => id).sort((a, b) => a - b)).toEqual([datedJan, undatedFeb]) + expect(exported.summary.total_containers).toBe(2) + }) + }) + describe('POST /api/export/containers/validate-studies', () => { it('returns valid and invalid study codes', async () => { await createTestStudy(ctx.db, { title: 'Valid', shortCode: 'VALID' }) From 16483fc7d998d940edb161aa8546b57a89eea1e2 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:53:06 -0700 Subject: [PATCH 08/28] refactor(statistics): select dashboard statistics through Container selection. getDashboardStatistics now asks selectContainers for its Containers and Specimens. created_from/created_to map to Specimen created and specimen_type_id to a one-item list. The inline all-Tags intersection, the unreachable location- and tag-only branch, the statistics location helpers and buildDateFilter are gone. The location filter no longer needs the raw SQLite handle, so the statistics routes drop it. The wire shape is unchanged. A filter that matches no Specimen returns the all-zero statistics, without the storage summary, as an unknown Study or Tag already did. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/index.ts | 2 +- .../__tests__/dashboard-stats.test.ts | 69 +++- .../__tests__/location-filter.test.ts | 122 ------ .../api/src/lib/statistics/dashboard-stats.ts | 352 ++++-------------- packages/api/src/lib/statistics/helpers.ts | 15 - .../api/src/lib/statistics/location-filter.ts | 92 ----- .../src/routes/__tests__/statistics.test.ts | 4 +- packages/api/src/routes/statistics.ts | 11 +- 8 files changed, 137 insertions(+), 530 deletions(-) delete mode 100644 packages/api/src/lib/statistics/__tests__/location-filter.test.ts delete mode 100644 packages/api/src/lib/statistics/helpers.ts delete mode 100644 packages/api/src/lib/statistics/location-filter.ts diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index c087a88..8545c0c 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -188,7 +188,7 @@ app.route('/api/strains', createStrainsRoutes(db)) app.route('/api/cell-lines', createCellLinesRoutes(db)) app.route('/api/plasmids', createPlasmidsRoutes(db)) app.route('/api/standards', createStandardsRoutes(db)) -app.route('/api/statistics', createStatisticsRoutes(db, sqlite)) +app.route('/api/statistics', createStatisticsRoutes(db)) app.route('/api/tags', createTagsRoutes(db)) app.route('/api/settings', createSettingsRoutes(db)) app.route('/api/units', createUnitsRoutes(db)) diff --git a/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts b/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts index bdc6e53..9cadc97 100644 --- a/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts +++ b/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts @@ -1,6 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import type { Database } from '../../../db/client' -import type { Database as SQLiteDatabase } from 'bun:sqlite' import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' import { createTestLocation, @@ -10,9 +9,13 @@ import { createTestSpecimen, createTestSpecimenType, createTestStudy, + createTestStudySubject, + createTestTag, + createTestUnit, } from '../../../__tests__/helpers/factories' import { utcNow } from '../../datetime' -import { micronixTube } from '../../../db/schema' +import { eq } from 'drizzle-orm' +import { micronixTube, specimen, storageContainerTag } from '../../../db/schema' import { getDashboardStatistics } from '../dashboard-stats' describe('dashboard-stats', () => { @@ -32,7 +35,7 @@ describe('dashboard-stats', () => { }) it('returns expected shape on empty database', async () => { - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, {}) + const data = await getDashboardStatistics(testDb, {}) expect(data.specimens).toMatchObject({ total: 0, @@ -54,7 +57,7 @@ describe('dashboard-stats', () => { }) it('returns empty results for unknown study short code', async () => { - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, { + const data = await getDashboardStatistics(testDb, { study: 'NONEXISTENT', }) @@ -68,8 +71,8 @@ describe('dashboard-stats', () => { const type = await createTestSpecimenType(testDb, { name: 'Blood' }) await createTestSpecimen(testDb, type.id) - const all = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, {}) - const unknown = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, { study: 'NONEXISTENT' }) + const all = await getDashboardStatistics(testDb, {}) + const unknown = await getDashboardStatistics(testDb, { study: 'NONEXISTENT' }) expect(all.specimens.total).toBe(1) expect(unknown.specimens.total).toBe(0) @@ -80,11 +83,57 @@ describe('dashboard-stats', () => { await createTestSpecimen(testDb, type.id) const today = utcNow().slice(0, 10) - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, { created_to: today }) + const data = await getDashboardStatistics(testDb, { created_to: today }) expect(data.specimens.total).toBe(1) }) + it('filters created_from and created_to on when the Specimen was created', async () => { + const type = await createTestSpecimenType(testDb, { name: 'Blood' }) + const older = await createTestSpecimen(testDb, type.id) + await createTestStorageContainer(testDb, { specimenId: older.id }) + await testDb.update(specimen).set({ created: '2020-06-01T00:00:00.000Z' }).where(eq(specimen.id, older.id)) + await createTestSpecimen(testDb, type.id) + + const data = await getDashboardStatistics(testDb, { created_from: '2020-01-01', created_to: '2020-12-31' }) + + expect(data.specimens.total).toBe(1) + expect(data.containers.total).toBe(1) + }) + + it('counts only Specimens with a Container carrying every selected Tag', async () => { + const type = await createTestSpecimenType(testDb, { name: 'Blood' }) + const red = await createTestTag(testDb, { name: 'red' }) + const blue = await createTestTag(testDb, { name: 'blue' }) + const unitId = (await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' })).id + const both = await createTestStorageContainer(testDb, { specimenId: (await createTestSpecimen(testDb, type.id)).id, unitId }) + const redOnly = await createTestStorageContainer(testDb, { specimenId: (await createTestSpecimen(testDb, type.id)).id, unitId }) + await testDb.insert(storageContainerTag).values([ + { storageContainerId: both.id, tagId: red.id }, + { storageContainerId: both.id, tagId: blue.id }, + { storageContainerId: redOnly.id, tagId: red.id }, + ]) + + const data = await getDashboardStatistics(testDb, { tag_ids: [red.id, blue.id] }) + + expect(data.specimens.total).toBe(1) + expect(data.containers.total).toBe(1) + expect(data.containers.averagePerSpecimen).toBe(1) + }) + + it('matches nothing for control Specimens within a Study', async () => { + const study = await createTestStudy(testDb, { title: 'Study', shortCode: 'ST' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'S1' }) + const type = await createTestSpecimenType(testDb, { name: 'Blood' }) + await createTestSpecimen(testDb, type.id, { studySubjectId: subject.id }) + + const subjects = await getDashboardStatistics(testDb, { study: 'ST', source_type: 'subject' }) + const controls = await getDashboardStatistics(testDb, { study: 'ST', source_type: 'control' }) + + expect(subjects.specimens.total).toBe(1) + expect(controls.specimens.total).toBe(0) + }) + it('includes containers at a filtered location', async () => { const storageType = await createTestStorageType(testDb, { name: 'Stats Storage' }) const loc = await createTestLocation(testDb, { @@ -100,7 +149,7 @@ describe('dashboard-stats', () => { position: 'B01', }) - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, { + const data = await getDashboardStatistics(testDb, { location_id: String(loc.id), }) @@ -111,7 +160,7 @@ describe('dashboard-stats', () => { it('returns byStatus for containers with remaining quantity', async () => { const container = await createTestStorageContainer(testDb) - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, {}) + const data = await getDashboardStatistics(testDb, {}) expect(data.containers.byStatus['In Use']).toBe(1) expect(data.containers.total).toBe(1) @@ -134,7 +183,7 @@ describe('dashboard-stats', () => { } await createTestSpecimen(testDb, type.id) // in another study - const data = await getDashboardStatistics(testDb, sqlite as SQLiteDatabase, { study: 'BIG' }) + const data = await getDashboardStatistics(testDb, { study: 'BIG' }) expect(data.specimens.total).toBe(3) expect(data.specimens.byStudy).toEqual({ BIG: 3 }) diff --git a/packages/api/src/lib/statistics/__tests__/location-filter.test.ts b/packages/api/src/lib/statistics/__tests__/location-filter.test.ts deleted file mode 100644 index f7f52bb..0000000 --- a/packages/api/src/lib/statistics/__tests__/location-filter.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import type { Database } from '../../../db/client' -import type { Database as SQLiteDatabase } from 'bun:sqlite' -import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' -import { - createTestBag, - createTestBox, - createTestLocation, - createTestMicronixPlate, - createTestMicronixTube, - createTestPaper, - createTestSheet, - createTestStaticWell, - createTestStorageContainer, - createTestStorageType, -} from '../../../__tests__/helpers/factories' -import { - resolveContainerIdsAtLocations, - resolveStatisticsLocationFilter, -} from '../location-filter' - -describe('location-filter', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) { - cleanupTestDatabase(sqlite) - } - }) - - it('resolveStatisticsLocationFilter returns none without location_id', async () => { - const result = await resolveStatisticsLocationFilter( - testDb, - sqlite as SQLiteDatabase, - undefined, - ) - expect(result).toEqual({ kind: 'none' }) - }) - - it('resolveStatisticsLocationFilter returns not_found for missing location', async () => { - const result = await resolveStatisticsLocationFilter( - testDb, - sqlite as SQLiteDatabase, - '99999', - ) - expect(result).toEqual({ kind: 'not_found' }) - }) - - it('resolveStatisticsLocationFilter returns resolved ids for an existing location', async () => { - const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) - const loc = await createTestLocation(testDb, { - name: 'Freezer A', - storageTypeId: String(storageType.id), - }) - - const result = await resolveStatisticsLocationFilter( - testDb, - sqlite as SQLiteDatabase, - String(loc.id), - ) - - expect(result).toEqual({ - kind: 'resolved', - filteredLocationIds: [loc.id], - }) - }) - - it('resolveContainerIdsAtLocations returns micronix container ids at matching locations', async () => { - const storageType = await createTestStorageType(testDb, { name: 'Shelf Storage' }) - const loc = await createTestLocation(testDb, { - name: 'Shelf 1', - storageTypeId: String(storageType.id), - }) - const otherStorageType = await createTestStorageType(testDb, { name: 'Other Storage' }) - const otherLoc = await createTestLocation(testDb, { - name: 'Shelf 2', - storageTypeId: String(otherStorageType.id), - }) - const plate = await createTestMicronixPlate(testDb, { name: 'Plate 1', locationId: loc.id }) - await createTestMicronixPlate(testDb, { name: 'Plate 2', locationId: otherLoc.id }) - - const container = await createTestStorageContainer(testDb) - await createTestMicronixTube(testDb, { - containerId: container.id, - collectionId: plate.id, - barcode: 'MX-LOC-1', - position: 'A01', - }) - - const ids = await resolveContainerIdsAtLocations(testDb, [loc.id]) - - expect(ids).toEqual([container.id]) - }) - - it('resolveContainerIdsAtLocations includes static wells and papers in boxes and bags', async () => { - const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) - const loc = await createTestLocation(testDb, { name: 'Freezer A', storageTypeId: String(storageType.id) }) - const plate = await createTestMicronixPlate(testDb, { name: 'P1', locationId: loc.id }) - const well = await createTestStorageContainer(testDb) - await createTestStaticWell(testDb, { containerId: well.id, collectionId: plate.id, position: 'A01' }) - const boxRow = await createTestBox(testDb, { name: 'Box1', locationId: loc.id }) - const bagRow = await createTestBag(testDb, { name: 'Bag1', locationId: loc.id }) - const boxSheet = await createTestSheet(testDb, { name: 'S1', boxId: boxRow.id }) - const bagSheet = await createTestSheet(testDb, { name: 'S1', bagId: bagRow.id }) - const reuse = { specimenId: well.specimenId, unitId: well.unitId! } - const paperInBox = await createTestStorageContainer(testDb, reuse) - const paperInBag = await createTestStorageContainer(testDb, reuse) - await createTestPaper(testDb, { containerId: paperInBox.id, sheetId: boxSheet.id }) - await createTestPaper(testDb, { containerId: paperInBag.id, sheetId: bagSheet.id }) - - const ids = await resolveContainerIdsAtLocations(testDb, [loc.id]) - - expect(ids.sort((a, b) => a - b)).toEqual([well.id, paperInBox.id, paperInBag.id].sort((a, b) => a - b)) - }) -}) diff --git a/packages/api/src/lib/statistics/dashboard-stats.ts b/packages/api/src/lib/statistics/dashboard-stats.ts index a455a4c..2db5e84 100644 --- a/packages/api/src/lib/statistics/dashboard-stats.ts +++ b/packages/api/src/lib/statistics/dashboard-stats.ts @@ -1,23 +1,8 @@ +import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' -import type { Database as SQLiteDatabase } from 'bun:sqlite' -import { - specimen, - storageContainer, - studySubject, - study, - specimenType, - tag, - storageContainerTag, -} from '../../db/schema' -import { eq, and, sql, inArray } from 'drizzle-orm' -import { resolveContainerTypes } from '../container-placement' -import { buildDateFilter } from './helpers' -import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' +import { study } from '../../db/schema' +import { selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '../container-selection' import { computeContainerAggregates } from './container-aggregates' -import { - resolveContainerIdsAtLocations, - resolveStatisticsLocationFilter, -} from './location-filter' import { computeSpecimenAggregates } from './specimen-aggregates' import { computeStorageStatistics } from './storage-aggregates' import type { DashboardStatistics, StatisticsFilters } from './types' @@ -47,272 +32,79 @@ function emptyDashboardStatistics(): DashboardStatistics { } } +/** Parse an integer id query value; anything else means no filter. */ +function parseId(value: string | undefined): number | undefined { + if (!value) return undefined + const id = parseInt(value) + return isNaN(id) ? undefined : id +} + +/** Map the statistics query to Container selection filters (ADR 0011). */ +function selectionFilters(filters: StatisticsFilters): ContainerSelectionFilters { + const specimenTypeId = parseId(filters.specimen_type_id) + return { + specimenTypeIds: specimenTypeId === undefined ? undefined : [specimenTypeId], + collectionDate: { from: filters.collection_date_from, to: filters.collection_date_to }, + specimenCreated: { from: filters.created_from, to: filters.created_to }, + containerTypes: filters.container_type ? [filters.container_type] : undefined, + tagIds: filters.tag_ids, + locationId: parseId(filters.location_id), + } +} + +/** + * The selection the statistics query describes, or null when it matches nothing: an + * unknown Study, or control Specimens within a Study. An unknown Location or Tag + * matches nothing inside the selection. + */ +async function statisticsSelection( + database: Database, + filters: StatisticsFilters, +): Promise { + const sourceType = + filters.source_type === 'subject' || filters.source_type === 'control' ? filters.source_type : undefined + if (!filters.study) return { kind: 'database', sourceType, filters: selectionFilters(filters) } + if (sourceType === 'control') return null + const studyRecord = await database + .select({ id: study.id }) + .from(study) + .where(eq(study.shortCode, filters.study)) + .get() + if (!studyRecord) return null + return { kind: 'study', studyId: studyRecord.id, filters: selectionFilters(filters) } +} + /** Filtered dashboard statistics for specimens, containers, and storage. */ export async function getDashboardStatistics( database: Database, - sqliteDatabase: SQLiteDatabase, filters: StatisticsFilters, ): Promise { - const { - study: studyCode, - source_type: sourceType, - specimen_type_id: specimenTypeId, - container_type: containerType, - tag_ids: tagIds, - location_id: locationId, - collection_date_from: collectionDateFrom, - collection_date_to: collectionDateTo, - created_from: createdFrom, - created_to: createdTo, - } = filters - - // Build specimen filter conditions - const specimenConditions = [] - - // Filter by study, selecting its subjects with a subquery rather than an id list - if (studyCode) { - const studyRecord = await database - .select() - .from(study) - .where(eq(study.shortCode, studyCode)) - .get() - - // An unknown study matches nothing; it must not fall through to whole-database totals. - if (!studyRecord) { - return emptyDashboardStatistics() - } - const studySubjectIds = database - .select({ id: studySubject.id }) - .from(studySubject) - .where(eq(studySubject.studyId, studyRecord.id)) - const anySubject = await database - .select({ id: studySubject.id }) - .from(studySubject) - .where(eq(studySubject.studyId, studyRecord.id)) - .limit(1) - .get() - if (!anySubject) { - // No subjects in study, return empty results - return emptyDashboardStatistics() - } - specimenConditions.push(inArray(specimen.studySubjectId, studySubjectIds)) - } - - if (sourceType === 'subject') { - specimenConditions.push(sql`${specimen.studySubjectId} IS NOT NULL`) - } else if (sourceType === 'control') { - specimenConditions.push(sql`${specimen.controlBatchId} IS NOT NULL`) - } - - if (specimenTypeId) { - const typeId = parseInt(specimenTypeId) - if (!isNaN(typeId)) { - specimenConditions.push(eq(specimen.specimenTypeId, typeId)) - } - } - - // Date filters for specimens - const collectionDateFilters = buildDateFilter(specimen.collectionDate, collectionDateFrom, collectionDateTo) - const createdDateFilters = buildDateFilter(specimen.created, createdFrom, createdTo) - specimenConditions.push(...collectionDateFilters, ...createdDateFilters) - - // Get filtered specimens - let specimenQuery = database.select().from(specimen) - if (specimenConditions.length > 0) { - specimenQuery = specimenQuery.where(and(...specimenConditions) as any) as any - } - const filteredSpecimens = await specimenQuery - - const specimenIds = filteredSpecimens.map(s => s.id) - - let specimenAggregates = await computeSpecimenAggregates(database, filteredSpecimens, { - useSpecimenTypeCache: true, - }) - - // Location Filtering - Query matching location IDs and their descendants - const hasLocationFilter = !!locationId - const locationFilter = await resolveStatisticsLocationFilter(database, sqliteDatabase, locationId) - let filteredLocationIds: number[] = [] - - if (locationFilter.kind === 'not_found') { - return emptyDashboardStatistics() - } + const selection = await statisticsSelection(database, filters) + if (!selection) return emptyDashboardStatistics() + const { containers, specimens, containerTypes } = await selectContainers(database, selection) + if (specimens.length === 0) return emptyDashboardStatistics() + + const hasContainerFilter = + !!filters.container_type || (filters.tag_ids?.length ?? 0) > 0 || parseId(filters.location_id) !== undefined + const specimenAggregates = await computeSpecimenAggregates(database, specimens, { + useSpecimenTypeCache: !hasContainerFilter, + }) + const containerTotal = containers.length + const averagePerSpecimen = specimenAggregates.total > 0 ? containerTotal / specimenAggregates.total : 0 + const containerAggregates = await computeContainerAggregates(database, containers, containerTypes) + const storage = await computeStorageStatistics( + database, + containers.map((container) => container.id), + containerTotal, + ) - if (locationFilter.kind === 'resolved') { - filteredLocationIds = locationFilter.filteredLocationIds - } - - // Get container IDs filtered by tags if tag filter is provided - // Using AND logic: containers must have ALL selected tags - let tagFilteredContainerIds: number[] | null = null - if (tagIds && tagIds.length > 0) { - - // For AND logic, find containers that have all selected tags - // Query containers for each tag and find the intersection - const containerSets: Set[] = [] - - for (const tagId of tagIds) { - const containersWithTag = await database - .select({ containerId: storageContainerTag.storageContainerId }) - .from(storageContainerTag) - .where(eq(storageContainerTag.tagId, tagId)) - containerSets.push(new Set(containersWithTag.map(ct => ct.containerId))) - } - - // Find intersection: containers that appear in all sets - if (containerSets.length > 0) { - let intersection = containerSets[0] - for (let i = 1; i < containerSets.length; i++) { - intersection = new Set([...intersection].filter(id => containerSets[i].has(id))) - } - tagFilteredContainerIds = Array.from(intersection) - } else { - tagFilteredContainerIds = [] - } - - if (tagFilteredContainerIds.length === 0) { - // No containers match all the tags, return empty results - return emptyDashboardStatistics() - } - } - - // Container Statistics - // Use efficient joins when location filter is present - let filteredContainers: Array = [] - - if (specimenIds.length > 0) { - // Get location-filtered container IDs once (if location filter is active) - let locationFilteredContainerIds: number[] | null = null - if (hasLocationFilter && filteredLocationIds.length > 0) { - locationFilteredContainerIds = await resolveContainerIdsAtLocations( - database, - filteredLocationIds, - tagFilteredContainerIds, - ) - } - - const locationFilteredIdSet = locationFilteredContainerIds ? new Set(locationFilteredContainerIds) : null - const tagFilteredIdSet = tagFilteredContainerIds ? new Set(tagFilteredContainerIds) : null - - // Batch query if too many specimen IDs to avoid SQLite variable limit - // Use Promise.all to parallelize queries for better performance - const specimenChunks = chunkArray(specimenIds, SQL_IN_CHUNK_SIZE) - - const chunkResults = await Promise.all( - specimenChunks.map(async (specimenChunk, chunkIndex) => { - const containers = await database - .select() - .from(storageContainer) - .where(inArray(storageContainer.specimenId, specimenChunk)) - // Location and tag filters are applied in memory: as IN lists they can exceed - // SQLite's parameter limit on large locations. - return containers.filter( - (c) => - (!locationFilteredIdSet || locationFilteredIdSet.has(c.id)) && - (!tagFilteredIdSet || tagFilteredIdSet.has(c.id)), - ) - }) - ) - - // Flatten results - filteredContainers = chunkResults.flat() - - // Remove duplicates (in case a container appears in multiple chunks - shouldn't happen, but be safe) - const seenIds = new Set() - filteredContainers = filteredContainers.filter(c => { - if (seenIds.has(c.id)) { - return false - } - seenIds.add(c.id) - return true - }) - } else if (filteredSpecimens.length === 0) { - // No specimens match, so no containers - return emptyDashboardStatistics() - } else { - // No specimen filter, but we might have state/status/location filters - if (hasLocationFilter && filteredLocationIds.length > 0) { - const locationFilteredContainerIds = await resolveContainerIdsAtLocations( - database, - filteredLocationIds, - tagFilteredContainerIds, - ) - - if (locationFilteredContainerIds.length === 0) { - filteredContainers = [] - } else { - // Fetch containers matching location and other filters - // Use Promise.all to parallelize queries - const containerIdChunks = chunkArray(locationFilteredContainerIds, SQL_IN_CHUNK_SIZE) - const chunkResults = await Promise.all( - containerIdChunks.map(async (containerIdChunk) => { - // Tag filter is already intersected into locationFilteredContainerIds. - return await database - .select() - .from(storageContainer) - .where(inArray(storageContainer.id, containerIdChunk)) - }) - ) - filteredContainers = chunkResults.flat() - } - } else { - // No location filter - if (tagFilteredContainerIds) { - const tagIdChunks = chunkArray(tagFilteredContainerIds, SQL_IN_CHUNK_SIZE) - const chunkResults = await Promise.all( - tagIdChunks.map((idChunk) => - database.select().from(storageContainer).where(inArray(storageContainer.id, idChunk)), - ), - ) - filteredContainers = chunkResults.flat() - } else { - filteredContainers = await database.select().from(storageContainer) - } - } - } - - // Get container types in batch - const containerIds = filteredContainers.map(c => c.id) - const containerTypeMap = await resolveContainerTypes(database, containerIds) - - // Filter by container type if specified (before calculating totals) - let finalContainers = filteredContainers - if (containerType) { - finalContainers = filteredContainers.filter(container => - containerTypeMap.get(container.id) === containerType - ) - } - - const containerTotal = finalContainers.length - const finalContainerIds = finalContainers.map((c) => c.id) - - if (containerType || tagFilteredContainerIds || hasLocationFilter) { - const filteredSpecimenIds = new Set(finalContainers.map(c => c.specimenId)) - const adjustedSpecimens = filteredSpecimens.filter((s) => filteredSpecimenIds.has(s.id)) - - specimenAggregates = await computeSpecimenAggregates(database, adjustedSpecimens, { - useSpecimenTypeCache: false, - }) - } - - const averagePerSpecimen = - specimenAggregates.total > 0 ? containerTotal / specimenAggregates.total : 0 - - const containerAggregates = await computeContainerAggregates( - database, - finalContainers, - containerTypeMap, - ) - - const storage = await computeStorageStatistics(database, finalContainerIds, containerTotal) - - return { - specimens: specimenAggregates, - containers: { - total: containerTotal, - ...containerAggregates, - averagePerSpecimen: Math.round(averagePerSpecimen * 100) / 100, - }, - storage, - } + return { + specimens: specimenAggregates, + containers: { + total: containerTotal, + ...containerAggregates, + averagePerSpecimen: Math.round(averagePerSpecimen * 100) / 100, + }, + storage, + } } diff --git a/packages/api/src/lib/statistics/helpers.ts b/packages/api/src/lib/statistics/helpers.ts deleted file mode 100644 index 1b7e3a1..0000000 --- a/packages/api/src/lib/statistics/helpers.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { gte } from 'drizzle-orm' -import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' -import { dateToUpperBound } from '../sql-helpers' - -/** Build date filter conditions for specimen date columns. */ -export function buildDateFilter(column: SQLiteColumn, dateFrom?: string, dateTo?: string) { - const conditions: ReturnType[] = [] - if (dateFrom) { - conditions.push(gte(column, dateFrom)) - } - if (dateTo) { - conditions.push(dateToUpperBound(column, dateTo)) - } - return conditions -} diff --git a/packages/api/src/lib/statistics/location-filter.ts b/packages/api/src/lib/statistics/location-filter.ts deleted file mode 100644 index 9a83f43..0000000 --- a/packages/api/src/lib/statistics/location-filter.ts +++ /dev/null @@ -1,92 +0,0 @@ -import type { Database } from '../../db/client' -import type { Database as SQLiteDatabase } from 'bun:sqlite' -import { - location, - micronixPlate, - micronixTube, - staticWell, - cryovialBox, - cryovialTube, - box, - bag, - sheet, - paper, -} from '../../db/schema' -import { eq, inArray, or } from 'drizzle-orm' - -export type StatisticsLocationFilterResult = - | { kind: 'none' } - | { kind: 'not_found' } - | { kind: 'resolved'; filteredLocationIds: number[] } - -/** Resolve a location_id filter to the target location and its descendants. */ -export async function resolveStatisticsLocationFilter( - database: Database, - sqliteDatabase: SQLiteDatabase, - locationId?: string, -): Promise { - if (!locationId) { - return { kind: 'none' } - } - - const id = parseInt(locationId) - if (isNaN(id)) { - return { kind: 'none' } - } - - const targetLocation = await database.select().from(location).where(eq(location.id, id)).get() - if (!targetLocation) { - return { kind: 'not_found' } - } - - const { getLocationDescendants } = await import('../location-helpers') - const descendants = await getLocationDescendants(sqliteDatabase, id) - - return { - kind: 'resolved', - filteredLocationIds: [id, ...descendants.map((d) => d.id)], - } -} - -/** - * Resolve storage container ids stored in the given locations: tubes and static wells on - * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags. - * Collections are matched with subqueries so large locations never build huge IN lists. - */ -export async function resolveContainerIdsAtLocations( - database: Database, - filteredLocationIds: number[], - tagFilteredContainerIds?: number[] | null, -): Promise { - const platesHere = database - .select({ id: micronixPlate.id }) - .from(micronixPlate) - .where(inArray(micronixPlate.locationId, filteredLocationIds)) - const cryovialBoxesHere = database - .select({ id: cryovialBox.id }) - .from(cryovialBox) - .where(inArray(cryovialBox.locationId, filteredLocationIds)) - const sheetsHere = database - .select({ id: sheet.id }) - .from(sheet) - .where( - or( - inArray(sheet.boxId, database.select({ id: box.id }).from(box).where(inArray(box.locationId, filteredLocationIds))), - inArray(sheet.bagId, database.select({ id: bag.id }).from(bag).where(inArray(bag.locationId, filteredLocationIds))), - ), - ) - - const [micronix, wells, cryovial, papers] = await Promise.all([ - database.select({ id: micronixTube.id }).from(micronixTube).where(inArray(micronixTube.collectionId, platesHere)), - database.select({ id: staticWell.id }).from(staticWell).where(inArray(staticWell.collectionId, platesHere)), - database.select({ id: cryovialTube.id }).from(cryovialTube).where(inArray(cryovialTube.collectionId, cryovialBoxesHere)), - database.select({ id: paper.id }).from(paper).where(inArray(paper.sheetId, sheetsHere)), - ]) - - const ids = new Set([...micronix, ...wells, ...cryovial, ...papers].map((r) => r.id)) - if (tagFilteredContainerIds) { - const tagged = new Set(tagFilteredContainerIds) - return [...ids].filter((id) => tagged.has(id)) - } - return [...ids] -} diff --git a/packages/api/src/routes/__tests__/statistics.test.ts b/packages/api/src/routes/__tests__/statistics.test.ts index 2f20d71..e927c04 100644 --- a/packages/api/src/routes/__tests__/statistics.test.ts +++ b/packages/api/src/routes/__tests__/statistics.test.ts @@ -17,8 +17,8 @@ describe('Statistics API', () => { password: 'password123', role: 'admin', }, - mount: (app, { db, sqlite }) => { - app.route('/api/statistics', createStatisticsRoutes(db, sqlite)) + mount: (app, { db }) => { + app.route('/api/statistics', createStatisticsRoutes(db)) }, }) }) diff --git a/packages/api/src/routes/statistics.ts b/packages/api/src/routes/statistics.ts index 54c3043..6648ec5 100644 --- a/packages/api/src/routes/statistics.ts +++ b/packages/api/src/routes/statistics.ts @@ -1,18 +1,13 @@ import { Hono } from 'hono' import type { Database } from '../db/client' -import type { Database as SQLiteDatabase } from 'bun:sqlite' import { getAdminStatistics } from '../lib/statistics/admin-stats' import { getDashboardStatistics } from '../lib/statistics/dashboard-stats' import { parseStatisticsFilters } from '../lib/statistics/parse-filters' import { handleRouteError } from '../lib/error-handler' import { createAdminMiddleware, createAuthMiddleware } from '../middleware/auth' -/** - * Create statistics routes with database injection - * @param database - Database instance (required) - * @param sqliteDatabase - Raw SQLite database instance (required for location queries) - */ -export function createStatisticsRoutes(database: Database, sqliteDatabase: SQLiteDatabase) { +/** Create statistics routes with database injection. */ +export function createStatisticsRoutes(database: Database) { const statistics = new Hono() const authMiddleware = createAuthMiddleware(database) const adminMiddleware = createAdminMiddleware(database) @@ -20,7 +15,7 @@ export function createStatisticsRoutes(database: Database, sqliteDatabase: SQLit statistics.get('/', authMiddleware, async (c) => { try { const filters = parseStatisticsFilters(c) - const data = await getDashboardStatistics(database, sqliteDatabase, filters) + const data = await getDashboardStatistics(database, filters) return c.json(data) } catch (error: unknown) { return handleRouteError(error, c) From ac28c650994686cab667d5a24c4893c33fcc2fca Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 16:54:09 -0700 Subject: [PATCH 09/28] refactor(export): select multi-study and barcode exports through Container selection. A multi-study upload is now one subject-list selection instead of one buildContainerQuery call per Study, followed by a container type filter and enrichment per Study. The executor groups the enriched rows by Study for the Export summary. The barcode export selects the found tubes with a micronix barcode selection and applies no other filters; found and not found still come from resolveMicronixBarcodesToContainers, since the selection carries no barcodes. buildMultiStudyContainerQuery, buildContainerQueryByMicronixBarcodes and MultiStudyExportResult had no other callers and are deleted. The multi-visit test moves to the executor tests. Co-Authored-By: Claude Opus 5.5 --- .../export/__tests__/container-export.test.ts | 22 ++ .../src/lib/export/__tests__/query.test.ts | 76 ----- .../api/src/lib/export/container-export.ts | 263 ++++++++++++------ packages/api/src/lib/export/query.ts | 253 +---------------- packages/api/src/lib/export/types.ts | 6 - 5 files changed, 203 insertions(+), 417 deletions(-) diff --git a/packages/api/src/lib/export/__tests__/container-export.test.ts b/packages/api/src/lib/export/__tests__/container-export.test.ts index 5c2a169..6185eea 100644 --- a/packages/api/src/lib/export/__tests__/container-export.test.ts +++ b/packages/api/src/lib/export/__tests__/container-export.test.ts @@ -191,6 +191,28 @@ describe('Container export executors', () => { expect(containerIds(result.data)).toEqual([alphaJan.id]) }) + it('keeps every listed visit of a Subject', async () => { + const study = await createTestStudy(db, { title: 'Visits', shortCode: 'VIS' }) + const subject = await createTestStudySubject(db, { studyId: study.id, name: 'S01' }) + const jan = await seedTube((await seedSpecimen({ studySubjectId: subject.id }, '2024-01-01')).id) + const feb = await seedTube((await seedSpecimen({ studySubjectId: subject.id }, '2024-02-03')).id) + await seedTube((await seedSpecimen({ studySubjectId: subject.id }, '2024-03-01')).id) + + const result = await runMultiStudyContainerExportPost(db, { + entries: [ + { study_short_code: 'VIS', subject_name: 'S01', collection_date: '2024-01-01' }, + { study_short_code: 'VIS', subject_name: 'S01', collection_date: '2024-02-01' }, + ], + filters: {}, + format: 'json', + columns, + dateTolerance: 2, + }) + + if (!('data' in result)) throw new Error('expected an export envelope') + expect(containerIds(result.data)).toEqual([jan.id, feb.id]) + }) + it('counts with the same summary as the export', async () => { await seedTwoStudies() diff --git a/packages/api/src/lib/export/__tests__/query.test.ts b/packages/api/src/lib/export/__tests__/query.test.ts index f12399e..58bcd93 100644 --- a/packages/api/src/lib/export/__tests__/query.test.ts +++ b/packages/api/src/lib/export/__tests__/query.test.ts @@ -15,9 +15,7 @@ import { utcNow } from '../../datetime' import { eq } from 'drizzle-orm' import { buildContainerQuery, - buildMultiStudyContainerQuery, resolveMicronixBarcodesToContainers, - buildContainerQueryByMicronixBarcodes, } from '../query' describe('buildContainerQuery', () => { @@ -212,77 +210,3 @@ describe('resolveMicronixBarcodesToContainers', () => { expect(result.size).toBe(0) }) }) - -describe('buildContainerQueryByMicronixBarcodes', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) cleanupTestDatabase(sqlite) - }) - - it('returns empty result for empty container ids', async () => { - const result = await buildContainerQueryByMicronixBarcodes(testDb, []) - expect(result.containers).toHaveLength(0) - expect(result.specimens).toHaveLength(0) - expect(result.studies).toHaveLength(0) - expect(result.subjectToStudyMap.size).toBe(0) - }) -}) - - -describe('buildMultiStudyContainerQuery subject dates', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) cleanupTestDatabase(sqlite) - }) - - it('keeps every listed visit per subject and includes subjects listed without a date', async () => { - const study = await createTestStudy(testDb, { title: 'Visits', shortCode: 'VIS', leadPerson: 'L' }) - const s01 = await createTestStudySubject(testDb, { studyId: study.id, name: 'S01' }) - const s02 = await createTestStudySubject(testDb, { studyId: study.id, name: 'S02' }) - const type = await createTestSpecimenType(testDb, { name: 'Blood' }) - const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) - const now = utcNow() - const addSpecimen = async (subjectId: number, collectionDate: string) => { - const [row] = await testDb - .insert(specimenTable) - .values({ studySubjectId: subjectId, specimenTypeId: type.id, collectionDate, created: now, lastUpdated: now }) - .returning() - await createTestStorageContainer(testDb, { specimenId: row.id, unitId: unit.id }) - return row - } - const jan = await addSpecimen(s01.id, '2024-01-01') - const feb = await addSpecimen(s01.id, '2024-02-01') - await addSpecimen(s01.id, '2024-03-01') - const may = await addSpecimen(s02.id, '2024-05-05') - - const result = await buildMultiStudyContainerQuery( - testDb, - [ - { study_short_code: 'VIS', subject_name: 'S01', collection_date: '2024-01-01' }, - { study_short_code: 'VIS', subject_name: 'S01', collection_date: '2024-02-01' }, - { study_short_code: 'VIS', subject_name: 'S02' }, - ], - {}, - ) - - expect(result.containers.map((c) => c.specimen_id).sort((a, b) => a - b)).toEqual( - [jan.id, feb.id, may.id].sort((a, b) => a - b), - ) - }) -}) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index ede5bc5..e109cdb 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,12 +1,24 @@ -import type { ContainerExportData } from '@sampledb/contract' +import { subjectDateSelections, type ContainerExportData } from '@sampledb/contract' +import { eq, inArray } from 'drizzle-orm' import type { Database } from '../../db/client' +import { study, studySubject } from '../../db/schema' import { ExpectedNotFoundError, RouteError } from '../error-handler' import { enrichContainerData } from './enrich' import { filterContainerIdsByType } from './filter' import { formatAsCSV, formatAsExcel, formatAsJSON } from './format' -import { buildContainerQuery, buildContainerQueryByMicronixBarcodes, buildMultiStudyContainerQuery, resolveMicronixBarcodesToContainers } from './query' +import { buildContainerQuery, resolveMicronixBarcodesToContainers } from './query' import { buildExportSummary, validateStudyCodes } from './validate' -import { resolveSubjectNamesByStudy, resolveStudyByShortCode } from '../identifier-resolution' +import { + resolveSubjectNamesByStudy, + resolveStudyByShortCode, + resolveSubjectsByStudyGrouped, +} from '../identifier-resolution' +import { + selectContainers, + type ContainerSelectionFilters, + type SubjectSelectionEntry, +} from '../container-selection' +import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' import type { CSVExportOptions, ExportFilters, @@ -316,6 +328,33 @@ export async function runSingleStudyContainerExportPost( }) } +/** Placeholder study for export metadata when an export spans Studies or has none (ADR 0011). */ +const MULTI_STUDY_RECORD: StudyRecord = { + id: 0, + shortCode: 'MULTI', + title: 'Multi-Study Export', + description: null, + leadPerson: '', + isLongitudinal: false, + created: '', + lastUpdated: '', + createdBy: null, + updatedBy: null, +} + +/** Container selection filters for an export's filters; `created_*` means Container created (ADR 0011). */ +function exportSelectionFilters( + filters: Omit +): ContainerSelectionFilters { + return { + specimenTypeIds: filters.specimen_type_ids, + collectionDate: { from: filters.date_from, to: filters.date_to }, + containerCreated: { from: filters.created_from, to: filters.created_to }, + containerTypes: filters.container_types, + tagIds: filters.tag_ids, + } +} + export async function runMultiStudyContainerExportPost( database: Database, params: { @@ -330,45 +369,140 @@ export async function runMultiStudyContainerExportPost( } ): Promise { const { entries, filters, format, columns, csvOptions, userId, countOnly, dateTolerance = 0 } = params - const result = await buildMultiStudyContainerQuery(database, entries, filters, dateTolerance) + const filenameBase = `multi_study_export_${formatExportTimestamp()}` - if (countOnly) { - return { count: result.containers.length, summary: result.summary } + const validation = await validateStudyCodes( + database, + entries.map((entry) => entry.study_short_code) + ) + if (validation.invalid.length > 0) { + const summary: MultiStudyExportSummary = { + total_containers: 0, + studies: [], + invalid_study_codes: validation.invalid, + errors: [`Invalid study codes: ${validation.invalid.join(', ')}`], + } + if (countOnly) return { count: 0, summary } + const formatted = await formatContainerExport( + database, + [], + format, + { study: 'MULTI' }, + MULTI_STUDY_RECORD, + columns, + csvOptions, + userId + ) + return buildPostExportEnvelope({ format, filenameBase, summary, ...formatted }) } - const firstStudy = - result.studies.size > 0 - ? Array.from(result.studies.values())[0] - : ({ - id: 0, - shortCode: 'MULTI', - title: 'Multi-Study Export', - description: null, - leadPerson: '', - isLongitudinal: false, - created: '', - lastUpdated: '', - createdBy: null, - updatedBy: null, - } as StudyRecord) + const entriesByStudy = new Map() + for (const entry of entries) { + const studyId = validation.valid.get(entry.study_short_code)! + entriesByStudy.set(studyId, [...(entriesByStudy.get(studyId) ?? []), entry]) + } + const subjectsByStudy = await resolveSubjectsByStudyGrouped( + database, + entries.map((entry) => ({ + studyId: validation.valid.get(entry.study_short_code)!, + subjectName: entry.subject_name, + })) + ) + + // The whole upload is one subject-list selection. A Subject listed without a date matches + // every collection date. + const subjects: SubjectSelectionEntry[] = [] + const subjectToStudyMap = new Map() + for (const [studyId, studyEntries] of entriesByStudy) { + const subjectIds = subjectsByStudy.get(studyId) ?? new Map() + for (const [subjectName, selection] of subjectDateSelections(studyEntries)) { + const subjectId = subjectIds.get(subjectName) + if (subjectId === undefined) continue + subjects.push({ studyId, subjectId, dates: selection.kind === 'dates' ? selection.filters : undefined }) + subjectToStudyMap.set(subjectId, validation.studies.get(studyId)!) + } + } + + const selected = + subjects.length > 0 + ? await selectContainers(database, { + kind: 'subjects', + subjects, + dateTolerance, + filters: exportSelectionFilters(filters), + }) + : { containers: [], specimens: [] } + const enrichedData = await enrichContainerData( + database, + selected.containers, + selected.specimens, + MULTI_STUDY_RECORD, + undefined, + subjectToStudyMap + ) + + // The Export summary groups the one selection by Study. + const studySummaries: MultiStudyExportSummary['studies'] = [] + for (const [studyId, studyEntries] of entriesByStudy) { + const studyRecord = validation.studies.get(studyId)! + const subjectNameToId = subjectsByStudy.get(studyId) ?? new Map() + const subjectIdToName = new Map([...subjectNameToId].map(([name, id]) => [id, name])) + const studyData = enrichedData.filter((row) => row.study_id === studyId) + const studySummary = await buildExportSummary( + studyData, + studyEntries.map((entry) => entry.subject_name), + subjectNameToId, + subjectIdToName + ) + studySummaries.push({ + study_code: studyRecord.shortCode, + study_title: studyRecord.title, + study_lead_person: studyRecord.leadPerson, + containers: studyData.length, + subjects_with_results: studySummary.subjects_with_results, + subjects_no_results: studySummary.subjects_no_results, + subjects_not_found: studySummary.subjects_not_found, + }) + } + const summary: MultiStudyExportSummary = { + total_containers: enrichedData.length, + studies: studySummaries, + invalid_study_codes: [], + } + + if (countOnly) { + return { count: enrichedData.length, summary } + } const formatted = await formatContainerExport( database, - result.containers, + enrichedData, format, { study: 'MULTI' }, - firstStudy, + validation.studies.values().next().value ?? MULTI_STUDY_RECORD, columns, csvOptions, userId ) - return buildPostExportEnvelope({ - format, - filenameBase: `multi_study_export_${formatExportTimestamp()}`, - summary: result.summary, - ...formatted, - }) + return buildPostExportEnvelope({ format, filenameBase, summary, ...formatted }) +} + +/** Each Subject's Study, by Subject id. */ +async function loadStudiesBySubject( + database: Database, + subjectIds: number[] +): Promise> { + const studiesBySubject = new Map() + for (const chunk of chunkArray([...new Set(subjectIds)], SQL_IN_CHUNK_SIZE)) { + const rows = await database + .select({ subjectId: studySubject.id, study }) + .from(studySubject) + .innerJoin(study, eq(studySubject.studyId, study.id)) + .where(inArray(studySubject.id, chunk)) + for (const row of rows) studiesBySubject.set(row.subjectId, row.study) + } + return studiesBySubject } export async function runBarcodeContainerExportPost( @@ -382,73 +516,30 @@ export async function runBarcodeContainerExportPost( } ): Promise { const { barcodes, format, columns, csvOptions, userId } = params + // Selected Containers carry no barcode, so found and not found come from the tubes. const barcodeToContainerId = await resolveMicronixBarcodesToContainers(database, barcodes) - const containerIds = Array.from(barcodeToContainerId.values()) const foundBarcodes = Array.from(barcodeToContainerId.keys()) - const notFoundBarcodes = barcodes.filter((b) => !barcodeToContainerId.has(b)) - - const summary = { - total_containers: 0, - barcodes_found: foundBarcodes, - barcodes_not_found: notFoundBarcodes, - } - - if (containerIds.length === 0) { - return buildPostExportEnvelope({ - format, - filenameBase: `barcode_export_${formatExportTimestamp()}`, - summary, - jsonData: [], - csv: '', - xlsx: Buffer.from(''), - }) - } - - const queryResult = await buildContainerQueryByMicronixBarcodes(database, containerIds) - if (queryResult.containers.length === 0) { - return buildPostExportEnvelope({ - format, - filenameBase: `barcode_export_${formatExportTimestamp()}`, - summary, - jsonData: [], - csv: '', - xlsx: Buffer.from(''), - }) - } - - const firstStudy = - queryResult.studies.length > 0 - ? queryResult.studies[0] - : ({ - id: 0, - shortCode: 'MULTI', - title: 'Multi-Study Export', - description: null, - leadPerson: '', - isLongitudinal: false, - created: '', - lastUpdated: '', - createdBy: null, - updatedBy: null, - } as StudyRecord) + const selected = await selectContainers(database, { kind: 'micronixBarcodes', barcodes: foundBarcodes }) + const studiesBySubject = await loadStudiesBySubject( + database, + selected.specimens.flatMap((row) => (row.studySubjectId == null ? [] : [row.studySubjectId])) + ) const enrichedData = await enrichContainerData( database, - queryResult.containers, - queryResult.specimens, - firstStudy, + selected.containers, + selected.specimens, + MULTI_STUDY_RECORD, undefined, - queryResult.subjectToStudyMap + studiesBySubject ) - summary.total_containers = enrichedData.length - const formatted = await formatContainerExport( database, enrichedData, format, { study: 'MULTI' }, - firstStudy, + studiesBySubject.values().next().value ?? MULTI_STUDY_RECORD, columns, csvOptions, userId @@ -457,7 +548,11 @@ export async function runBarcodeContainerExportPost( return buildPostExportEnvelope({ format, filenameBase: `barcode_export_${formatExportTimestamp()}`, - summary, + summary: { + total_containers: enrichedData.length, + barcodes_found: foundBarcodes, + barcodes_not_found: barcodes.filter((barcode) => !barcodeToContainerId.has(barcode)), + }, ...formatted, }) } diff --git a/packages/api/src/lib/export/query.ts b/packages/api/src/lib/export/query.ts index 5ff12d0..0679873 100644 --- a/packages/api/src/lib/export/query.ts +++ b/packages/api/src/lib/export/query.ts @@ -7,20 +7,9 @@ import { micronixTube, } from '../../db/schema' import { eq, and, or, inArray, gte, lte, sql, type SQL } from 'drizzle-orm' -import { resolveSubjectsByStudyGrouped } from '../identifier-resolution' import { resolveContainerIdsWithAllTags } from '../container-tag-filter' -import { enrichContainerData } from './enrich' -import { filterContainerIdsByType } from './filter' -import { subjectDateSelections, type SubjectDateFilter, type SubjectDateSelection } from '@sampledb/contract' -import type { - ContainerExportData, - ExportFilters, - MultiStudyExportEntry, - MultiStudyExportResult, - MultiStudyExportSummary, - StudyRecord, -} from './types' -import { buildExportSummary, validateStudyCodes } from './validate' +import type { SubjectDateFilter } from '@sampledb/contract' +import type { ExportFilters } from './types' import { chunkArray, dateToUpperBound, SQL_IN_CHUNK_SIZE } from '../sql-helpers' import { shiftCalendarDate } from '../datetime' @@ -187,84 +176,6 @@ export async function resolveMicronixBarcodesToContainers( return result } -/** - * Build container query by micronix container IDs (multi-study support) - */ -export async function buildContainerQueryByMicronixBarcodes( - database: Database, - containerIds: number[] -): Promise<{ - containers: Array - specimens: Array<{ id: number; studySubjectId: number | null; controlBatchId: number | null; specimenTypeId: number; collectionDate: string | null; created: string }> - studies: StudyRecord[] - subjectToStudyMap: Map -}> { - if (containerIds.length === 0) { - return { containers: [], specimens: [], studies: [], subjectToStudyMap: new Map() } - } - - // Get containers - const containers = await database - .select() - .from(storageContainer) - .where(inArray(storageContainer.id, containerIds)) - - if (containers.length === 0) { - return { containers: [], specimens: [], studies: [], subjectToStudyMap: new Map() } - } - - // Get specimens for these containers - // specimenId can be null from DB; filter to numbers only - const specimenIds = [...new Set(containers.map(c => c.specimenId).filter((id): id is number => typeof id === 'number'))] - if (specimenIds.length === 0) { - return { containers, specimens: [], studies: [], subjectToStudyMap: new Map() } - } - - const specimens = await database - .select({ - id: specimen.id, - studySubjectId: specimen.studySubjectId, - controlBatchId: specimen.controlBatchId, - specimenTypeId: specimen.specimenTypeId, - collectionDate: specimen.collectionDate, - created: specimen.created, - }) - .from(specimen) - .where(inArray(specimen.id, specimenIds)) - - // Get unique study IDs from subjects - const subjectIds = [...new Set(specimens.filter(s => s.studySubjectId !== null).map(s => s.studySubjectId!))] - const studies: StudyRecord[] = [] - const subjectToStudyMap = new Map() - - if (subjectIds.length > 0) { - const subjects = await database - .select({ id: studySubject.id, studyId: studySubject.studyId }) - .from(studySubject) - .where(inArray(studySubject.id, subjectIds)) - - const uniqueStudyIds = [...new Set(subjects.map(s => s.studyId))] - if (uniqueStudyIds.length > 0) { - const studyRecords = await database - .select() - .from(study) - .where(inArray(study.id, uniqueStudyIds)) - studies.push(...studyRecords) - - // Build subject to study map - const studyMap = new Map(studyRecords.map(s => [s.id, s])) - for (const subject of subjects) { - const studyRecord = studyMap.get(subject.studyId) - if (studyRecord) { - subjectToStudyMap.set(subject.id, studyRecord) - } - } - } - } - - return { containers, specimens, studies, subjectToStudyMap } -} - /** Collection date condition for one per-subject filter (exact date +/- tolerance days, or a range). */ function subjectDateCondition(dateFilter: SubjectDateFilter, toleranceDays: number) { if ('exact' in dateFilter) { @@ -282,163 +193,3 @@ function subjectDateCondition(dateFilter: SubjectDateFilter, toleranceDays: numb ] return bounds.length > 0 ? and(...bounds) : null } - -export async function buildMultiStudyContainerQuery( - database: Database, - entries: MultiStudyExportEntry[], - filters: Omit, - dateTolerance: number = 0 -): Promise { - // Validate all study codes - const studyCodes = entries.map(e => e.study_short_code) - const validation = await validateStudyCodes(database, studyCodes) - - if (validation.invalid.length > 0) { - // Return early with invalid study codes - return { - containers: [], - studies: validation.studies, - summary: { - total_containers: 0, - studies: [], - invalid_study_codes: validation.invalid, - errors: [`Invalid study codes: ${validation.invalid.join(', ')}`], - }, - } - } - - // Group entries by study - const entriesByStudy = new Map() - for (const entry of entries) { - const studyId = validation.valid.get(entry.study_short_code) - if (studyId) { - if (!entriesByStudy.has(studyId)) { - entriesByStudy.set(studyId, []) - } - entriesByStudy.get(studyId)!.push(entry) - } - } - - // Resolve subjects for each study - const subjectResolutionEntries: Array<{ studyId: number; subjectName: string }> = [] - for (const [studyId, studyEntries] of entriesByStudy.entries()) { - for (const entry of studyEntries) { - subjectResolutionEntries.push({ studyId, subjectName: entry.subject_name }) - } - } - - const subjectsByStudy = await resolveSubjectsByStudyGrouped(database, subjectResolutionEntries) - - // Date selection per subject name, for studies where at least one subject is limited to some dates - const subjectDatesByStudy = new Map>() - for (const [studyId, studyEntries] of entriesByStudy.entries()) { - const selections = subjectDateSelections(studyEntries) - if ([...selections.values()].some((selection) => selection.kind === 'dates')) { - subjectDatesByStudy.set(studyId, selections) - } - } - - // Process each study separately - const allContainers: ContainerExportData[] = [] - const studySummaries: MultiStudyExportSummary['studies'] = [] - - for (const [studyId, studyEntries] of entriesByStudy.entries()) { - const studyRecord = validation.studies.get(studyId)! - const studyCode = studyRecord.shortCode - const subjectMap = subjectsByStudy.get(studyId) || new Map() - const subjectIds = Array.from(subjectMap.values()) - - if (subjectIds.length === 0) { - // No subjects found for this study - studySummaries.push({ - study_code: studyCode, - study_title: studyRecord.title, - study_lead_person: studyRecord.leadPerson, - containers: 0, - subjects_with_results: [], - subjects_no_results: [], - subjects_not_found: studyEntries.map(e => e.subject_name), - }) - continue - } - - // Build filters for this study - const studyFilters: ExportFilters = { - study: studyCode, - subject_ids: subjectIds, - date_tolerance: dateTolerance, - ...filters, - } - - // Add subject dates if available - const studySubjectDates = subjectDatesByStudy.get(studyId) - if (studySubjectDates && studySubjectDates.size > 0) { - studyFilters.subject_dates = {} - for (const [subjectName, selection] of studySubjectDates.entries()) { - const subjectId = subjectMap.get(subjectName) - // Subjects with no entry match every date. - if (subjectId && selection.kind === 'dates') { - studyFilters.subject_dates[subjectId] = selection.filters - } - } - } - - // Query containers for this study - const { containers, specimens } = await buildContainerQuery(database, studyFilters) - - // Apply container type filter if specified - let filteredContainers = containers - if (filters.container_types && filters.container_types.length > 0) { - const containerIds = containers.map(c => c.id) - const matchingIds = await filterContainerIdsByType(database, containerIds, filters.container_types) - filteredContainers = containers.filter(c => matchingIds.includes(c.id)) - } - - // Enrich container data - const enrichedData = await enrichContainerData( - database, - filteredContainers, - specimens || [], - studyRecord, - filters.container_types, - undefined - ) - - allContainers.push(...enrichedData) - - // Build summary for this study - const subjectIdToName = new Map() - for (const [name, id] of subjectMap.entries()) { - subjectIdToName.set(id, name) - } - - const studySummary = await buildExportSummary( - enrichedData, - studyEntries.map(e => e.subject_name), - subjectMap, - subjectIdToName - ) - - studySummaries.push({ - study_code: studyCode, - study_title: studyRecord.title, - study_lead_person: studyRecord.leadPerson, - containers: enrichedData.length, - subjects_with_results: studySummary.subjects_with_results, - subjects_no_results: studySummary.subjects_no_results, - subjects_not_found: studySummary.subjects_not_found, - }) - } - - return { - containers: allContainers, - studies: validation.studies, - summary: { - total_containers: allContainers.length, - studies: studySummaries, - invalid_study_codes: validation.invalid, - }, - } -} - - diff --git a/packages/api/src/lib/export/types.ts b/packages/api/src/lib/export/types.ts index 9d73324..1f89325 100644 --- a/packages/api/src/lib/export/types.ts +++ b/packages/api/src/lib/export/types.ts @@ -22,12 +22,6 @@ export interface MultiStudyExportEntry { date_to?: string } -export interface MultiStudyExportResult { - containers: import('@sampledb/contract').ContainerExportData[] - studies: Map - summary: MultiStudyExportSummary -} - export interface MultiStudyExportSummary { total_containers: number studies: Array<{ From 57cd93a5b39e3e7004440c1f671cf288fed6dc54 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:08:01 -0700 Subject: [PATCH 10/28] refactor(export): take export study fields from the Source and delete the old selection code. enrichContainerData now takes only the Containers. Each subject row's study fields come from its Source's Study (ADR 0007) and control rows have none, so the type filter, the unused specimen list, the study argument and the subject-to-Study map are gone, along with loadStudiesBySubject. The MULTI study record is now used only for JSON export metadata, including the barcode export's. buildContainerQuery, filterContainerIdsByType (lib/export/filter.ts) and lib/container-tag-filter.ts had no callers outside their tests and are deleted with them; Container selection's tests cover their cases. query.ts keeps resolveMicronixBarcodesToContainers. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/container-tag-filter.ts | 32 --- .../src/lib/export/__tests__/enrich.test.ts | 262 +++--------------- .../src/lib/export/__tests__/filter.test.ts | 88 ------ .../src/lib/export/__tests__/query.test.ts | 180 +----------- .../api/src/lib/export/container-export.ts | 54 +--- packages/api/src/lib/export/enrich.ts | 40 +-- packages/api/src/lib/export/filter.ts | 41 --- packages/api/src/lib/export/query.ts | 170 +----------- 8 files changed, 66 insertions(+), 801 deletions(-) delete mode 100644 packages/api/src/lib/container-tag-filter.ts delete mode 100644 packages/api/src/lib/export/__tests__/filter.test.ts delete mode 100644 packages/api/src/lib/export/filter.ts diff --git a/packages/api/src/lib/container-tag-filter.ts b/packages/api/src/lib/container-tag-filter.ts deleted file mode 100644 index c02007c..0000000 --- a/packages/api/src/lib/container-tag-filter.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { eq } from 'drizzle-orm' -import type { Database } from '../db/client' -import { storageContainerTag } from '../db/schema' - -/** - * Resolve container IDs that have ALL of the given tags (AND semantics). - * Returns empty array when no containers match. - */ -export async function resolveContainerIdsWithAllTags( - database: Database, - tagIds: number[], -): Promise { - if (tagIds.length === 0) return [] - - const containerSets: Set[] = [] - for (const tagId of tagIds) { - const rows = await database - .select({ containerId: storageContainerTag.storageContainerId }) - .from(storageContainerTag) - .where(eq(storageContainerTag.tagId, tagId)) - containerSets.push(new Set(rows.map((row) => row.containerId))) - } - - if (containerSets.length === 0) return [] - - let intersection = containerSets[0] - for (let i = 1; i < containerSets.length; i++) { - intersection = new Set([...intersection].filter((id) => containerSets[i].has(id))) - } - - return Array.from(intersection) -} diff --git a/packages/api/src/lib/export/__tests__/enrich.test.ts b/packages/api/src/lib/export/__tests__/enrich.test.ts index 2ec3b4e..bed9c17 100644 --- a/packages/api/src/lib/export/__tests__/enrich.test.ts +++ b/packages/api/src/lib/export/__tests__/enrich.test.ts @@ -16,12 +16,9 @@ import { createTestTag, } from '../../../__tests__/helpers/factories' import type { Database } from '../../../db/client' -import { micronixTube, cryovialTube, cryovialBox, storageContainerTag, box, sheet, paper } from '../../../db/schema' -import { utcNow } from '../../datetime' +import { micronixTube, storageContainerTag, box, sheet, paper } from '../../../db/schema' import { utcNow } from '../../datetime' import { enrichContainerData } from '../enrich' -import { buildContainerQuery } from '../query' -import type { StudyRecord } from '../types' describe('enrichContainerData', () => { let testDb: Database @@ -78,21 +75,7 @@ describe('enrichContainerData', () => { const plate = await createTestMicronixPlate(testDb, { name: 'Plate-1', locationId: loc.id }) const container = await createMicronixContainer(testDb, spec.id, plate, 'MT-001', 'A01') - const enriched = await enrichContainerData( - testDb, - [container], - [ - { - id: spec.id, - studySubjectId: subject.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: '2024-06-01', - created: spec.created, - }, - ], - study as StudyRecord, - ) + const enriched = await enrichContainerData(testDb, [container]) expect(enriched).toHaveLength(1) expect(enriched[0]).toMatchObject({ @@ -125,10 +108,6 @@ describe('enrichContainerData', () => { const batch = await createTestControlBatch(testDb, definition.id, { name: 'Batch-June' }) const specimenType = await createTestSpecimenType(testDb, { name: 'Control blood' }) const spec = await createTestSpecimen(testDb, specimenType.id, { controlBatchId: batch.id }) - const placeholderStudy = await createTestStudy(testDb, { - title: 'Unused', - shortCode: 'UNUSED', - }) const unit = await createTestUnit(testDb, { symbol: `uL-ctrl-${Date.now()}`, name: 'microliter', @@ -136,21 +115,7 @@ describe('enrichContainerData', () => { }) const container = await createTestStorageContainer(testDb, { specimenId: spec.id, unitId: unit.id }) - const enriched = await enrichContainerData( - testDb, - [container], - [ - { - id: spec.id, - studySubjectId: null, - controlBatchId: batch.id, - specimenTypeId: specimenType.id, - collectionDate: null, - created: spec.created, - }, - ], - placeholderStudy as StudyRecord, - ) + const enriched = await enrichContainerData(testDb, [container]) expect(enriched).toHaveLength(1) expect(enriched[0]).toMatchObject({ @@ -169,161 +134,50 @@ describe('enrichContainerData', () => { expect(enriched[0].strain_composition).toContain('100%') }) - it('filters out container types not in container_types', async () => { - const study = await createTestStudy(testDb, { - title: 'Filter Study', - shortCode: 'FILT', - }) - const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj' }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Serum' }) - const spec = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id }) - const storageType = await createTestStorageType(testDb, { name: 'Shelf' }) - const loc = await createTestLocation(testDb, { - name: 'Loc', - storageTypeId: String(storageType.id), - canContainCollections: true, - }) - const plate = await createTestMicronixPlate(testDb, { name: 'P-mic', locationId: loc.id }) - const micronix = await createMicronixContainer(testDb, spec.id, plate, 'MIC-1', 'A01') - - const unit = await createTestUnit(testDb, { - symbol: `uL-cv-${Date.now()}`, - name: 'microliter', - category: 'volume', - }) - const cryoContainer = await createTestStorageContainer(testDb, { specimenId: spec.id, unitId: unit.id }) - const now = utcNow() - const [box] = await testDb - .insert(cryovialBox) - .values({ name: 'CryoBox', locationId: loc.id, created: now, lastUpdated: now }) - .returning() - await testDb.insert(cryovialTube).values({ - id: cryoContainer.id, - collectionId: box!.id, - barcode: 'CRYO-1', - position: null, - }) - - const specimenRow = { - id: spec.id, - studySubjectId: subject.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: null, - created: spec.created, - } - - const enriched = await enrichContainerData( - testDb, - [micronix, cryoContainer], - [specimenRow], - study as StudyRecord, - ['micronix_tube'], - ) - - expect(enriched).toHaveLength(1) - expect(enriched[0].container_type).toBe('micronix_tube') - expect(enriched[0].container_id).toBe(micronix.id) - }) - - it('uses subjectToStudyMap for per-subject study fields in multi-study export', async () => { - const studyA = await createTestStudy(testDb, { - title: 'Study Alpha', - shortCode: 'ALPHA', - leadPerson: 'Alpha Lead', - }) - const studyB = await createTestStudy(testDb, { - title: 'Study Beta', - shortCode: 'BETA', - leadPerson: 'Beta Lead', - }) + it("takes each row's study fields from its Source's Study, and none for a control row", async () => { + const studyA = await createTestStudy(testDb, { title: 'Study Alpha', shortCode: 'ALPHA', leadPerson: 'Alpha Lead' }) + const studyB = await createTestStudy(testDb, { title: 'Study Beta', shortCode: 'BETA', leadPerson: 'Beta Lead' }) const subjectA = await createTestStudySubject(testDb, { studyId: studyA.id, name: 'A1' }) const subjectB = await createTestStudySubject(testDb, { studyId: studyB.id, name: 'B1' }) + const definition = await createTestControlDefinition(testDb, { name: 'Mix', controlType: 'blood' }) + const batch = await createTestControlBatch(testDb, definition.id, { name: 'Batch-1' }) const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) const specA = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subjectA.id }) const specB = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subjectB.id }) - const unit = await createTestUnit(testDb, { - symbol: `uL-ms-${Date.now()}`, - name: 'microliter', - category: 'volume', - }) - const containerA = await createTestStorageContainer(testDb, { specimenId: specA.id, unitId: unit.id }) - const containerB = await createTestStorageContainer(testDb, { specimenId: specB.id, unitId: unit.id }) - - const subjectToStudyMap = new Map([ - [subjectA.id, studyA as StudyRecord], - [subjectB.id, studyB as StudyRecord], - ]) - - const enriched = await enrichContainerData( - testDb, - [containerA, containerB], - [ - { - id: specA.id, - studySubjectId: subjectA.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: null, - created: specA.created, - }, - { - id: specB.id, - studySubjectId: subjectB.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: null, - created: specB.created, - }, - ], - studyA as StudyRecord, - undefined, - subjectToStudyMap, - ) - - expect(enriched).toHaveLength(2) - const rowA = enriched.find((r) => r.subject_name === 'A1') - const rowB = enriched.find((r) => r.subject_name === 'B1') - expect(rowA?.study_code).toBe('ALPHA') - expect(rowB?.study_code).toBe('BETA') - }) - - describe('pipeline with buildContainerQuery', () => { - it('enriches containers returned from buildContainerQuery', async () => { - const study = await createTestStudy(testDb, { - title: 'Pipeline Study', - shortCode: 'PIPE', - leadPerson: 'Pipe Lead', - }) - const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'PipeSubj' }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Whole blood' }) - const spec = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id }) - const storageType = await createTestStorageType(testDb, { name: 'Cold' }) - const loc = await createTestLocation(testDb, { - name: 'Rack 1', - storageTypeId: String(storageType.id), - canContainCollections: true, - }) - const plate = await createTestMicronixPlate(testDb, { name: 'PipePlate', locationId: loc.id }) - await createMicronixContainer(testDb, spec.id, plate, 'PIPE-01', 'B02') - - const { study: resolvedStudy, containers, specimens } = await buildContainerQuery(testDb, { - study: 'PIPE', - }) - const enriched = await enrichContainerData( - testDb, - containers, - specimens!, - resolvedStudy, - ) - - expect(enriched).toHaveLength(1) - expect(enriched[0]).toMatchObject({ - subject_name: 'PipeSubj', - study_code: 'PIPE', - specimen_type: 'Whole blood', - barcode: 'PIPE-01', - }) + const specControl = await createTestSpecimen(testDb, specimenType.id, { controlBatchId: batch.id }) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const containers = [ + await createTestStorageContainer(testDb, { specimenId: specA.id, unitId: unit.id }), + await createTestStorageContainer(testDb, { specimenId: specB.id, unitId: unit.id }), + await createTestStorageContainer(testDb, { specimenId: specControl.id, unitId: unit.id }), + ] + + const enriched = await enrichContainerData(testDb, containers) + + const studyFields = (row: (typeof enriched)[number] | undefined) => ({ + study_id: row?.study_id, + study_code: row?.study_code, + study_title: row?.study_title, + study_lead_person: row?.study_lead_person, + }) + expect(enriched).toHaveLength(3) + expect(studyFields(enriched.find((row) => row.subject_name === 'A1'))).toEqual({ + study_id: studyA.id, + study_code: 'ALPHA', + study_title: 'Study Alpha', + study_lead_person: 'Alpha Lead', + }) + expect(studyFields(enriched.find((row) => row.subject_name === 'B1'))).toEqual({ + study_id: studyB.id, + study_code: 'BETA', + study_title: 'Study Beta', + study_lead_person: 'Beta Lead', + }) + expect(studyFields(enriched.find((row) => row.control_batch_id === batch.id))).toEqual({ + study_id: undefined, + study_code: undefined, + study_title: undefined, + study_lead_person: undefined, }) }) @@ -359,21 +213,7 @@ describe('enrichContainerData', () => { sublabel: 'Spot-7', }) - const enriched = await enrichContainerData( - testDb, - [container], - [ - { - id: spec.id, - studySubjectId: subject.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: '2024-06-01', - created: spec.created, - }, - ], - study as StudyRecord, - ) + const enriched = await enrichContainerData(testDb, [container]) expect(enriched).toHaveLength(1) expect(enriched[0]).toMatchObject({ @@ -403,21 +243,7 @@ describe('enrichContainerData', () => { { storageContainerId: container.id, tagId: qcTag.id }, ]) - const enriched = await enrichContainerData( - testDb, - [container], - [ - { - id: spec.id, - studySubjectId: subject.id, - controlBatchId: null, - specimenTypeId: specimenType.id, - collectionDate: null, - created: spec.created, - }, - ], - study, - ) + const enriched = await enrichContainerData(testDb, [container]) expect(enriched).toHaveLength(1) expect(enriched[0].tags).toBe('Hold, QC') diff --git a/packages/api/src/lib/export/__tests__/filter.test.ts b/packages/api/src/lib/export/__tests__/filter.test.ts deleted file mode 100644 index 133c564..0000000 --- a/packages/api/src/lib/export/__tests__/filter.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' -import { - createTestLocation, - createTestMicronixPlate, - createTestSpecimen, - createTestSpecimenType, - createTestStorageType, - createTestUnit, -} from '../../../__tests__/helpers/factories' -import type { Database } from '../../../db/client' -import { filterContainerIdsByType } from '../filter' -import { micronixTube, storageContainer } from '../../../db/schema' -import { utcNow } from '../../datetime' - -describe('filterContainerIdsByType', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) cleanupTestDatabase(sqlite) - }) - - it('returns all container ids when no filter', async () => { - const ids = [1, 2, 3] - const result = await filterContainerIdsByType(testDb, ids) - expect(result).toEqual(ids) - }) - - it('returns all container ids when empty filter array', async () => { - const ids = [1, 2] - const result = await filterContainerIdsByType(testDb, ids, []) - expect(result).toEqual(ids) - }) - - it('returns empty array when filter specified but no matching containers', async () => { - const ids: number[] = [] - const result = await filterContainerIdsByType(testDb, ids, ['micronix_tube']) - expect(result).toEqual([]) - }) - - it('filters to micronix tubes via placement reads', async () => { - const unit = await createTestUnit(testDb, { - symbol: `uL-filter-${Date.now()}`, - name: 'microliter', - category: 'volume', - }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) - const specimen = await createTestSpecimen(testDb, specimenType.id) - const now = utcNow() - const [container] = await testDb - .insert(storageContainer) - .values({ - specimenId: specimen.id, - unitId: unit.id, - totalQuantity: 1, - remainingQuantity: 1, - created: now, - lastUpdated: now, - }) - .returning() - - const storageType = await createTestStorageType(testDb, { name: `Lab-${container!.id}` }) - const location = await createTestLocation(testDb, { - name: `Lab-${container!.id}`, - storageTypeId: String(storageType.id), - }) - const plate = await createTestMicronixPlate(testDb, { - name: `Plate-${container!.id}`, - locationId: location.id, - }) - await testDb.insert(micronixTube).values({ - id: container!.id, - collectionId: plate.id, - barcode: `MX-${container!.id}`, - position: 'A01', - }) - - const result = await filterContainerIdsByType(testDb, [container!.id, 99999], ['micronix_tube']) - expect(result).toEqual([container!.id]) - }) -}) diff --git a/packages/api/src/lib/export/__tests__/query.test.ts b/packages/api/src/lib/export/__tests__/query.test.ts index 58bcd93..3578002 100644 --- a/packages/api/src/lib/export/__tests__/query.test.ts +++ b/packages/api/src/lib/export/__tests__/query.test.ts @@ -1,185 +1,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' -import { - createTestStudy, - createTestStudySubject, - createTestSpecimenType, - createTestSpecimen, - createTestStorageContainer, - createTestTag, - createTestUnit, -} from '../../../__tests__/helpers/factories' import type { Database } from '../../../db/client' -import { storageContainerTag, specimen as specimenTable } from '../../../db/schema' -import { utcNow } from '../../datetime' -import { eq } from 'drizzle-orm' -import { - buildContainerQuery, - resolveMicronixBarcodesToContainers, -} from '../query' - -describe('buildContainerQuery', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) cleanupTestDatabase(sqlite) - }) - - it('returns containers and study for study with subject and specimen', async () => { - const study = await createTestStudy(testDb, { - title: 'Export Study', - shortCode: 'EX1', - leadPerson: 'Lead', - }) - const subject = await createTestStudySubject(testDb, { - studyId: study.id, - name: 'Subject1', - }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) - const specimen = await createTestSpecimen(testDb, specimenType.id, { - studySubjectId: subject.id, - }) - await createTestStorageContainer(testDb, { specimenId: specimen.id }) - - const result = await buildContainerQuery(testDb, { study: 'EX1' }) - expect(result.study.shortCode).toBe('EX1') - expect(result.containers).toHaveLength(1) - expect(result.specimens).toBeDefined() - expect(result.specimens!.length).toBe(1) - }) - - it('throws for unknown study short code', async () => { - await expect( - buildContainerQuery(testDb, { study: 'UNKNOWN' }) - ).rejects.toThrow('not found') - }) - - it('returns empty containers when study has no specimens', async () => { - const study = await createTestStudy(testDb, { - title: 'Empty Study', - shortCode: 'EMPTY2', - leadPerson: 'X', - }) - await createTestStudySubject(testDb, { studyId: study.id, name: 'S1' }) - const result = await buildContainerQuery(testDb, { study: 'EMPTY2' }) - expect(result.containers).toHaveLength(0) - expect(result.study.shortCode).toBe('EMPTY2') - }) - - it('returns empty containers when study has no subjects', async () => { - const study = await createTestStudy(testDb, { - title: 'Empty New Study', - shortCode: 'EMPTY_NEW', - leadPerson: 'X', - }) - // No subjects created - const result = await buildContainerQuery(testDb, { study: 'EMPTY_NEW' }) - expect(result.containers).toHaveLength(0) - expect(result.study.shortCode).toBe('EMPTY_NEW') - }) - - it('returns only containers that have all selected tags (AND)', async () => { - const study = await createTestStudy(testDb, { - title: 'Tag Filter Study', - shortCode: 'TF1', - leadPerson: 'Lead', - }) - const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj1' }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) - const spec = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id }) - - const unit = await createTestUnit(testDb, { - symbol: `uL-tag-${Date.now()}`, - name: 'microliter', - category: 'volume', - }) - const bothTagsContainer = await createTestStorageContainer(testDb, { specimenId: spec.id, unitId: unit.id }) - const oneTagContainer = await createTestStorageContainer(testDb, { specimenId: spec.id, unitId: unit.id }) - - const qcTag = await createTestTag(testDb, { name: 'QC' }) - const holdTag = await createTestTag(testDb, { name: 'Hold' }) - - await testDb.insert(storageContainerTag).values([ - { storageContainerId: bothTagsContainer.id, tagId: qcTag.id }, - { storageContainerId: bothTagsContainer.id, tagId: holdTag.id }, - { storageContainerId: oneTagContainer.id, tagId: qcTag.id }, - ]) - - const result = await buildContainerQuery(testDb, { - study: 'TF1', - tag_ids: [qcTag.id, holdTag.id], - }) - - expect(result.containers).toHaveLength(1) - expect(result.containers[0].id).toBe(bothTagsContainer.id) - }) -}) - -describe('buildContainerQuery for studies with more subjects than SQLite allows as bound parameters', () => { - let testDb: Database - let sqlite: Awaited>['sqlite'] - const count = 70_000 - - beforeEach(async () => { - const setup = await setupTestDatabase() - testDb = setup.db - sqlite = setup.sqlite - }) - - afterEach(() => { - if (sqlite) cleanupTestDatabase(sqlite) - }) - - /** 70,000 subjects; the first three have one dated specimen and container each. */ - async function seedBigStudy() { - const study = await createTestStudy(testDb, { title: 'Big Study', shortCode: 'BIG' }) - const type = await createTestSpecimenType(testDb, { name: 'DBS' }) - const subjectIds: number[] = [] - sqlite.exec('BEGIN') - const addSubject = sqlite.prepare( - "INSERT INTO study_subject (study_id, name, created, last_updated) VALUES (?, ?, '2024-01-01', '2024-01-01')", - ) - for (let i = 0; i < count; i++) { - subjectIds.push(Number(addSubject.run(study.id, `S${i}`).lastInsertRowid)) - } - sqlite.exec('COMMIT') - const unit = await createTestUnit(testDb, { symbol: 'spots', name: 'spots', category: 'count' }) - for (const [i, subjectId] of subjectIds.slice(0, 3).entries()) { - const spec = await createTestSpecimen(testDb, type.id, { studySubjectId: subjectId }) - await testDb.update(specimenTable).set({ collectionDate: `2024-0${i + 1}-01` }).where(eq(specimenTable.id, spec.id)) - await createTestStorageContainer(testDb, { specimenId: spec.id, unitId: unit.id }) - } - return subjectIds - } - - it('exports the whole study', async () => { - await seedBigStudy() - - const result = await buildContainerQuery(testDb, { study: 'BIG' }) - - expect(result.containers).toHaveLength(3) - }, 60_000) - - it('exports a listed subject set with per-subject dates', async () => { - const subjectIds = await seedBigStudy() - - const result = await buildContainerQuery(testDb, { - study: 'BIG', - subject_ids: subjectIds, - subject_dates: { [subjectIds[0]]: [{ exact: '2024-01-01' }], [subjectIds[1]]: [{ exact: '2030-01-01' }] }, - }) - - // Subject 0 matches its date, subject 1 does not, subject 2 is listed without a date. - expect(result.specimens?.map((s) => s.studySubjectId).sort((a, b) => a! - b!)).toEqual([subjectIds[0], subjectIds[2]]) - }, 60_000) -}) +import { resolveMicronixBarcodesToContainers } from '../query' describe('resolveMicronixBarcodesToContainers', () => { let testDb: Database diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index 20fa5dd..c874e0c 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,7 +1,7 @@ import { subjectDateSelections, type ContainerExportData, type SubjectDateFilter } from '@sampledb/contract' -import { eq, inArray } from 'drizzle-orm' +import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' -import { study, studySubject } from '../../db/schema' +import { study } from '../../db/schema' import { selectContainers, type ContainerSelection, @@ -14,7 +14,6 @@ import { formatAsCSV, formatAsExcel, formatAsJSON } from './format' import { resolveMicronixBarcodesToContainers } from './query' import { buildExportSummary, validateStudyCodes } from './validate' import { resolveSubjectNamesByStudy, resolveSubjectsByStudyGrouped } from '../identifier-resolution' -import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' import type { CSVExportOptions, ExportFilters, @@ -178,7 +177,7 @@ export async function runSingleStudyContainerExportGet( throw new RouteError(400, { error: 'Invalid format. Use csv, xlsx, or json.' }) } - const enrichedData = await enrichContainerData(database, result.containers, result.specimens, studyRecord) + const enrichedData = await enrichContainerData(database, result.containers) const timestamp = formatExportTimestamp() const filenameBase = `study_${studyRecord.shortCode}_export_${timestamp}` @@ -287,7 +286,7 @@ export async function runSingleStudyContainerExportPost( return { count: result.containers.length, summary } } - const enrichedData = await enrichContainerData(database, result.containers, result.specimens, studyRecord) + const enrichedData = await enrichContainerData(database, result.containers) const summary = await buildExportSummary( enrichedData, @@ -386,14 +385,12 @@ export async function runMultiStudyContainerExportPost( // The whole upload is one subject-list selection. A Subject listed without a date matches // every collection date. const subjects: SubjectSelectionEntry[] = [] - const subjectToStudyMap = new Map() for (const [studyId, studyEntries] of entriesByStudy) { const subjectIds = subjectsByStudy.get(studyId) ?? new Map() for (const [subjectName, selection] of subjectDateSelections(studyEntries)) { const subjectId = subjectIds.get(subjectName) if (subjectId === undefined) continue subjects.push({ studyId, subjectId, dates: selection.kind === 'dates' ? selection.filters : undefined }) - subjectToStudyMap.set(subjectId, validation.studies.get(studyId)!) } } @@ -405,15 +402,8 @@ export async function runMultiStudyContainerExportPost( dateTolerance, filters: exportSelectionFilters(filters), }) - : { containers: [], specimens: [] } - const enrichedData = await enrichContainerData( - database, - selected.containers, - selected.specimens, - MULTI_STUDY_RECORD, - undefined, - subjectToStudyMap - ) + : { containers: [] } + const enrichedData = await enrichContainerData(database, selected.containers) // The Export summary groups the one selection by Study. const studySummaries: MultiStudyExportSummary['studies'] = [] @@ -462,23 +452,6 @@ export async function runMultiStudyContainerExportPost( return buildPostExportEnvelope({ format, filenameBase, summary, ...formatted }) } -/** Each Subject's Study, by Subject id. */ -async function loadStudiesBySubject( - database: Database, - subjectIds: number[] -): Promise> { - const studiesBySubject = new Map() - for (const chunk of chunkArray([...new Set(subjectIds)], SQL_IN_CHUNK_SIZE)) { - const rows = await database - .select({ subjectId: studySubject.id, study }) - .from(studySubject) - .innerJoin(study, eq(studySubject.studyId, study.id)) - .where(inArray(studySubject.id, chunk)) - for (const row of rows) studiesBySubject.set(row.subjectId, row.study) - } - return studiesBySubject -} - export async function runBarcodeContainerExportPost( database: Database, params: { @@ -495,25 +468,14 @@ export async function runBarcodeContainerExportPost( const foundBarcodes = Array.from(barcodeToContainerId.keys()) const selected = await selectContainers(database, { kind: 'micronixBarcodes', barcodes: foundBarcodes }) - const studiesBySubject = await loadStudiesBySubject( - database, - selected.specimens.flatMap((row) => (row.studySubjectId == null ? [] : [row.studySubjectId])) - ) - const enrichedData = await enrichContainerData( - database, - selected.containers, - selected.specimens, - MULTI_STUDY_RECORD, - undefined, - studiesBySubject - ) + const enrichedData = await enrichContainerData(database, selected.containers) const formatted = await formatContainerExport( database, enrichedData, format, { study: 'MULTI' }, - studiesBySubject.values().next().value ?? MULTI_STUDY_RECORD, + MULTI_STUDY_RECORD, columns, csvOptions, userId diff --git a/packages/api/src/lib/export/enrich.ts b/packages/api/src/lib/export/enrich.ts index ba0db52..330a05e 100644 --- a/packages/api/src/lib/export/enrich.ts +++ b/packages/api/src/lib/export/enrich.ts @@ -5,15 +5,15 @@ import { projectContainerPlacementFields, } from '../container-projection' import { loadContainerReadViews } from '../container-read-view' -import type { ContainerExportData, StudyRecord } from './types' +import type { ContainerExportData } from './types' +/** + * Container export rows for the selected Containers. Study fields come from each + * Specimen's Source (ADR 0007); a control row has none. + */ export async function enrichContainerData( database: Database, - containers: StorageContainer[], - _specimens: Array<{ id: number; studySubjectId: number | null; controlBatchId: number | null; specimenTypeId: number; collectionDate: string | null; created: string }>, - study: StudyRecord, - containerTypeFilter?: string[], - subjectToStudyMap?: Map + containers: StorageContainer[] ): Promise { const views = await loadContainerReadViews(database, containers) const enriched: ContainerExportData[] = [] @@ -38,27 +38,9 @@ export async function enrichContainerData( const controlBatchId = spec.controlBatchId || undefined const source = view.source const controlSource = source?.type === 'control' ? source : undefined + const subjectStudy = source?.type === 'subject' ? source.study : undefined const subjectName = source?.name - if (containerTypeFilter && containerTypeFilter.length > 0 && !containerTypeFilter.includes(containerType)) { - continue - } - - let containerStudy = study - if (subjectToStudyMap && subjectId) { - const subjectStudy = subjectToStudyMap.get(subjectId) - if (subjectStudy) { - containerStudy = subjectStudy - } - } else if (controlBatchId) { - containerStudy = study - } - - const studyId = controlBatchId ? undefined : containerStudy.id - const studyTitle = controlBatchId ? undefined : containerStudy.title - const studyCode = controlBatchId ? undefined : containerStudy.shortCode - const studyLeadPerson = controlBatchId ? undefined : containerStudy.leadPerson - const tagNames = [...view.tags.map((t) => t.name)].sort((a, b) => a.localeCompare(b)) enriched.push({ @@ -89,10 +71,10 @@ export async function enrichContainerData( target_density: controlSource?.targetDensity ?? undefined, target_density_unit: controlSource?.targetDensityUnit ?? undefined, strain_composition: controlSource?.strainComposition ?? undefined, - study_id: studyId, - study_title: studyTitle, - study_code: studyCode, - study_lead_person: studyLeadPerson, + study_id: subjectStudy?.id, + study_title: subjectStudy?.title, + study_code: subjectStudy?.code, + study_lead_person: subjectStudy?.leadPerson, location_path: view.placement.locationPath, location_id: view.placement.location?.id, location_name: view.placement.location?.name, diff --git a/packages/api/src/lib/export/filter.ts b/packages/api/src/lib/export/filter.ts deleted file mode 100644 index 27fcbff..0000000 --- a/packages/api/src/lib/export/filter.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { Database } from '../../db/client' -import type { ContainerSubtype } from '../container-placement' -import { resolveContainerTypes } from '../container-placement' - -const EXPORTABLE_SUBTYPES: ContainerSubtype[] = [ - 'micronix_tube', - 'cryovial_tube', - 'paper', - 'static_well', -] - -/** Filter container IDs to those matching export container_types via placement reads. */ -export async function filterContainerIdsByType( - database: Database, - containerIds: number[], - containerTypeFilter?: string[], -): Promise { - if (!containerTypeFilter || containerTypeFilter.length === 0) { - return containerIds - } - - if (containerIds.length === 0) { - return [] - } - - const allowedTypes = new Set( - containerTypeFilter.filter((type): type is ContainerSubtype => - EXPORTABLE_SUBTYPES.includes(type as ContainerSubtype), - ), - ) - - if (allowedTypes.size === 0) { - return [] - } - - const typeMap = await resolveContainerTypes(database, containerIds) - return containerIds.filter((id) => { - const containerType = typeMap.get(id) - return containerType !== undefined && allowedTypes.has(containerType) - }) -} diff --git a/packages/api/src/lib/export/query.ts b/packages/api/src/lib/export/query.ts index 0679873..50c755a 100644 --- a/packages/api/src/lib/export/query.ts +++ b/packages/api/src/lib/export/query.ts @@ -1,154 +1,6 @@ +import { inArray } from 'drizzle-orm' import type { Database } from '../../db/client' -import { - storageContainer, - specimen, - studySubject, - study, - micronixTube, -} from '../../db/schema' -import { eq, and, or, inArray, gte, lte, sql, type SQL } from 'drizzle-orm' -import { resolveContainerIdsWithAllTags } from '../container-tag-filter' -import type { SubjectDateFilter } from '@sampledb/contract' -import type { ExportFilters } from './types' -import { chunkArray, dateToUpperBound, SQL_IN_CHUNK_SIZE } from '../sql-helpers' -import { shiftCalendarDate } from '../datetime' - -export async function buildContainerQuery(database: Database, filters: ExportFilters) { - // First, get the study and its subjects - const studyRecord = await database - .select() - .from(study) - .where(eq(study.shortCode, filters.study)) - .get() - - if (!studyRecord) { - throw new Error(`Study with short code '${filters.study}' not found`) - } - - // A subject list (or per-subject dates) needs the study's subject ids; a whole-study export - // selects them with a subquery instead of an id list. - const studySubjectIds = database - .select({ id: studySubject.id }) - .from(studySubject) - .where(eq(studySubject.studyId, studyRecord.id)) - - const subjectDates = - filters.subject_dates && Object.keys(filters.subject_dates).length > 0 ? filters.subject_dates : undefined - const listsSubjects = (filters.subject_ids && filters.subject_ids.length > 0) || subjectDates !== undefined - - let filteredSubjectIds: number[] = [] - if (listsSubjects) { - filteredSubjectIds = (await studySubjectIds).map((s) => s.id) - if (filters.subject_ids && filters.subject_ids.length > 0) { - const requested = new Set(filters.subject_ids) - filteredSubjectIds = filteredSubjectIds.filter((id) => requested.has(id)) - } - if (filteredSubjectIds.length === 0) { - return { containers: [], study: studyRecord, specimens: [] } - } - } - - const specimenConditions: SQL[] = [sql`${specimen.studySubjectId} IS NOT NULL`] - if (filters.specimen_type_ids && filters.specimen_type_ids.length > 0) { - specimenConditions.push(inArray(specimen.specimenTypeId, filters.specimen_type_ids)) - } - const globalDateConditions = [ - ...(filters.date_from ? [gte(specimen.collectionDate, filters.date_from)] : []), - ...(filters.date_to ? [lte(specimen.collectionDate, filters.date_to)] : []), - ] - - /** Subject (and collection date) condition for one chunk of listed subjects. */ - const subjectChunkCondition = (subjectIds: number[]): SQL | undefined => { - if (!subjectDates) return and(inArray(specimen.studySubjectId, subjectIds), ...globalDateConditions) - const tolerance = filters.date_tolerance || 0 - const conditions: SQL[] = [] - const undatedSubjectIds: number[] = [] - for (const subjectId of subjectIds) { - const entry = subjectDates[subjectId] as SubjectDateFilter | SubjectDateFilter[] | undefined - const dateConditions = (entry == null ? [] : Array.isArray(entry) ? entry : [entry]) - .map((dateFilter) => subjectDateCondition(dateFilter, tolerance)) - .filter((condition) => condition != null) - if (dateConditions.length === 0) { - // Listed without a date: every collection date for this subject. - undatedSubjectIds.push(subjectId) - continue - } - // Several entries for one subject (one per visit) are alternatives. - conditions.push(and(eq(specimen.studySubjectId, subjectId), or(...dateConditions))!) - } - if (undatedSubjectIds.length > 0) { - conditions.push(and(inArray(specimen.studySubjectId, undatedSubjectIds), ...globalDateConditions)!) - } - return or(...conditions) - } - - // Listed subjects are queried a chunk at a time, so neither the IN list nor the per-subject - // OR grows with the upload. - const subjectConditions = listsSubjects - ? chunkArray(filteredSubjectIds, SQL_IN_CHUNK_SIZE).map(subjectChunkCondition) - : [and(inArray(specimen.studySubjectId, studySubjectIds), ...globalDateConditions)] - - const specimens: Array<{ - id: number - studySubjectId: number | null - controlBatchId: number | null - specimenTypeId: number - collectionDate: string | null - created: string - }> = [] - for (const subjectCondition of subjectConditions) { - specimens.push( - ...(await database - .select({ - id: specimen.id, - studySubjectId: specimen.studySubjectId, - controlBatchId: specimen.controlBatchId, - specimenTypeId: specimen.specimenTypeId, - collectionDate: specimen.collectionDate, - created: specimen.created, - }) - .from(specimen) - .where(and(...specimenConditions, subjectCondition))), - ) - } - - const specimenIds = specimens.map(s => s.id) - if (specimenIds.length === 0) { - return { containers: [], study: studyRecord } - } - - // Build conditions for container query - const containerConditions: any[] = [] - - if (filters.created_from) { - containerConditions.push(gte(storageContainer.created, filters.created_from)) - } - - if (filters.created_to) { - containerConditions.push(dateToUpperBound(storageContainer.created, filters.created_to)) - } - - // Get matching containers, chunking specimen ids to stay under SQLite's parameter limit - let containers: Array = [] - for (const idChunk of chunkArray(specimenIds, SQL_IN_CHUNK_SIZE)) { - containers.push( - ...(await database - .select() - .from(storageContainer) - .where(and(inArray(storageContainer.specimenId, idChunk), ...containerConditions) as any)), - ) - } - - if (filters.tag_ids && filters.tag_ids.length > 0) { - const matchingIds = new Set(await resolveContainerIdsWithAllTags(database, filters.tag_ids)) - if (matchingIds.size === 0) { - return { containers: [], study: studyRecord, specimens: [] } - } - containers = containers.filter((container) => matchingIds.has(container.id)) - } - - return { containers, study: studyRecord, specimens } -} +import { micronixTube } from '../../db/schema' /** * Resolve micronix barcodes to container IDs @@ -175,21 +27,3 @@ export async function resolveMicronixBarcodesToContainers( return result } - -/** Collection date condition for one per-subject filter (exact date +/- tolerance days, or a range). */ -function subjectDateCondition(dateFilter: SubjectDateFilter, toleranceDays: number) { - if ('exact' in dateFilter) { - const from = shiftCalendarDate(dateFilter.exact, -toleranceDays) - const to = shiftCalendarDate(dateFilter.exact, toleranceDays) - if (!from || !to) return eq(specimen.collectionDate, dateFilter.exact) - return and( - gte(specimen.collectionDate, from), - lte(specimen.collectionDate, to), - ) - } - const bounds = [ - ...(dateFilter.from ? [gte(specimen.collectionDate, dateFilter.from)] : []), - ...(dateFilter.to ? [lte(specimen.collectionDate, dateFilter.to)] : []), - ] - return bounds.length > 0 ? and(...bounds) : null -} From edbf02341e914942925796fadc3b86880adb6fde Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:08:05 -0700 Subject: [PATCH 11/28] docs: point export and statistics code at Container selection. Every export and statistics path now selects through src/lib/container-selection/, so the rule lists no unmigrated paths. Co-Authored-By: Claude Opus 5.5 --- CODING_STANDARDS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CODING_STANDARDS.md b/CODING_STANDARDS.md index 1276613..ef8a487 100644 --- a/CODING_STANDARDS.md +++ b/CODING_STANDARDS.md @@ -31,6 +31,7 @@ Paths are relative to `packages/api/`. - Return the full `SpecimenSource` shape to every caller, not a per-endpoint subset (ADR 0007). - Load a container read payload (placement, Tags, Specimen, Source) with `loadContainerReadViews` or `loadContainerReadViewsByIds` from `src/lib/container-read-view.ts` (ADR 0009). - Project the loaded view to its read path's type: `EnrichedContainerApi` for list, derivations and PATCH, `EnrichedStorageContainer` for collection detail, and CSV row fields for Container export (ADR 0009). +- To choose which Containers an export or statistic covers, call `selectContainers` from `src/lib/container-selection/`, and filter a paginated Container query by Tags with `containerHasAllTags` (ADR 0011). ### Routes From 46deaf8172a14dd9a0dc33ca9047420d13b6e0b7 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:18:00 -0700 Subject: [PATCH 12/28] refactor(api): let Container selection own its filter rules. Container selection now owns the rules its callers re-derived: - containerTypes is typed as ContainerSubtype. containerTypesFilter narrows wire values and drops unrecognised ones, so a list of only unrecognised values still matches nothing; 'unknown' matches Containers with no subtype row, as ADR 0011 now says. - hasContainerLevelFilter replaces dashboard-stats' own rule, which left out Container created. - SpecimenSourceType and specimenSourceType replace the ternary in the database selection and the check in statisticsSelection. Export and statistics look up a Study through identifier-resolution (resolveStudyRecordByShortCode backs resolveStudyByShortCode) instead of their own queries. Export inverts subject name maps with one helper, and multi-study JSON metadata uses MULTI_STUDY_RECORD, as barcode export does. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0011-container-selection.md | 2 +- .../__tests__/container-selection.test.ts | 8 ++--- .../src/lib/container-selection/conditions.ts | 28 ++++++++++++++--- .../src/lib/container-selection/filters.ts | 25 +++++++++++++++ .../api/src/lib/container-selection/index.ts | 17 ++++------ .../api/src/lib/container-selection/types.ts | 13 ++++++-- .../api/src/lib/export/container-export.ts | 29 ++++++++--------- packages/api/src/lib/identifier-resolution.ts | 16 +++++----- .../api/src/lib/statistics/dashboard-stats.ts | 31 +++++++++---------- 9 files changed, 109 insertions(+), 60 deletions(-) create mode 100644 packages/api/src/lib/container-selection/filters.ts diff --git a/docs/adr/0011-container-selection.md b/docs/adr/0011-container-selection.md index 364f376..b081b5d 100644 --- a/docs/adr/0011-container-selection.md +++ b/docs/adr/0011-container-selection.md @@ -32,7 +32,7 @@ We put **Container selection** (see [CONTEXT.md](../../CONTEXT.md)) behind one m - **Tags**: a Container matches when it carries every selected Tag. This holds for every caller, `GET /containers` included. - **Created**: Specimen created and Container created are separate filters. The routes keep their wire names: export's `created_from`/`created_to` map to Container created, and statistics' to Specimen created. -- **Container type**: applied once, here. The GET export's "No containers found" 404 now fires on the filtered result, so a Study whose Containers are all of other types returns 404 instead of an empty file. +- **Container type**: applied once, here. The GET export's "No containers found" 404 now fires on the filtered result, so a Study whose Containers are all of other types returns 404 instead of an empty file. The type `unknown` matches Containers with no subtype row; any other unrecognised type matches nothing. - **Location**: the Location and every Location under it. - **Collection dates**: a listed Subject with its own dates ignores the global collection-date range. Undated Subjects get the global range. Tolerance widens exact dates only. - **Specimens**: with no Container-level filter (container type, Tag, Location, Container created), every matching Specimen is included, with or without Containers. With one, only Specimens that have a matching Container are included. Export ignores Specimens with no Container. diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index 05d3353..5fa49e4 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -31,7 +31,7 @@ import { storageContainer as storageContainerTable, storageContainerTag, } from '../../../db/schema' -import { containerHasAllTags, selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '..' +import { containerHasAllTags, containerTypesFilter, selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '..' describe('selectContainers', () => { let db: Database @@ -236,7 +236,7 @@ describe('selectContainers', () => { const untypedOnly = await seedSpecimen({ studySubjectId: subject.id }) const untyped = await seedContainer(untypedOnly.id, { kind: 'untyped' }) - const vials = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['cryovial_tube', 'not_a_type'] })) + const vials = await selectContainers(db, select(study.id, subject.id, { containerTypes: containerTypesFilter(['cryovial_tube', 'not_a_type']) })) const unknown = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['unknown'] })) expect(ids(vials.containers)).toEqual([vial.id]) @@ -246,12 +246,12 @@ describe('selectContainers', () => { expect(ids(unknown.specimens)).toEqual([untypedOnly.id]) }) - it('matches nothing when every listed container type is unknown', async () => { + it('matches nothing when every listed container type is unrecognised', async () => { const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) const subject = await seedSubject(study.id, 'P1') await seedContainer((await seedSpecimen({ studySubjectId: subject.id })).id) - const result = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['not_a_type'] })) + const result = await selectContainers(db, select(study.id, subject.id, { containerTypes: containerTypesFilter(['not_a_type']) })) expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) }) diff --git a/packages/api/src/lib/container-selection/conditions.ts b/packages/api/src/lib/container-selection/conditions.ts index 7af07de..be81a33 100644 --- a/packages/api/src/lib/container-selection/conditions.ts +++ b/packages/api/src/lib/container-selection/conditions.ts @@ -1,4 +1,4 @@ -import { count, gte, inArray, sql, type SQL } from 'drizzle-orm' +import { count, gte, inArray, isNotNull, sql, type SQL } from 'drizzle-orm' import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' import { bag, @@ -17,7 +17,7 @@ import { } from '../../db/schema' import type { ContainerSubtype } from '../container-placement' import { dateToUpperBound } from '../sql-helpers' -import type { ContainerSelectionFilters, DateRange } from './types' +import type { ContainerSelectionFilters, DateRange, SpecimenSourceType } from './types' /** * A Container's type, from the subtype table holding it, in the precedence @@ -39,6 +39,16 @@ export function dateRangeConditions(column: SQLiteColumn, range: DateRange | und ] } +/** Specimens of the given source type; undefined, every Specimen. */ +export function sourceTypeCondition(sourceType: SpecimenSourceType | undefined): SQL | undefined { + if (sourceType === undefined) return undefined + const sources: Record = { + subject: isNotNull(specimen.studySubjectId), + control: isNotNull(specimen.controlBatchId), + } + return sources[sourceType] +} + /** * Conditions on the Specimen row other than its collection date, which depends on the * selection: specimen types and Specimen created. @@ -58,8 +68,10 @@ export function specimenFilterConditions(filters: ContainerSelectionFilters): SQ */ export function containerFilterConditions(filters: ContainerSelectionFilters): SQL[] { const conditions = [...dateRangeConditions(storageContainer.created, filters.containerCreated)] - if (filters.containerTypes && filters.containerTypes.length > 0) { - conditions.push(inArray(containerTypeSql, [...new Set(filters.containerTypes)])) + if (filters.containerTypes) { + conditions.push( + filters.containerTypes.length > 0 ? inArray(containerTypeSql, [...new Set(filters.containerTypes)]) : sql`0`, + ) } const tags = containerHasAllTags(filters.tagIds ?? []) if (tags) conditions.push(tags) @@ -67,6 +79,14 @@ export function containerFilterConditions(filters: ContainerSelectionFilters): S return conditions } +/** + * Whether any Container-level filter is set: Container created, container types, Tags or + * Location. With one set, the selection keeps only Specimens with a matching Container. + */ +export function hasContainerLevelFilter(filters: ContainerSelectionFilters): boolean { + return containerFilterConditions(filters).length > 0 +} + /** * Containers stored in a Location or any Location under it: tubes and static wells on * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags there. diff --git a/packages/api/src/lib/container-selection/filters.ts b/packages/api/src/lib/container-selection/filters.ts new file mode 100644 index 0000000..4290279 --- /dev/null +++ b/packages/api/src/lib/container-selection/filters.ts @@ -0,0 +1,25 @@ +import type { ContainerSubtype } from '../container-placement' +import type { SpecimenSourceType } from './types' + +/** Every Container type, in display order. `'unknown'` is a Container with no subtype row. */ +export const CONTAINER_TYPES = [ + 'micronix_tube', + 'cryovial_tube', + 'paper', + 'static_well', + 'unknown', +] as const satisfies readonly ContainerSubtype[] + +/** + * Narrow wire container types to the Container types filter. No values means every type; + * unrecognised values are dropped, so a list of only unrecognised values matches nothing. + */ +export function containerTypesFilter(values: readonly string[] | undefined): ContainerSubtype[] | undefined { + if (!values || values.length === 0) return undefined + return CONTAINER_TYPES.filter((type) => values.includes(type)) +} + +/** Narrow a wire source type; anything other than `'subject'` or `'control'` means both. */ +export function specimenSourceType(value: string | undefined): SpecimenSourceType | undefined { + return value === 'subject' || value === 'control' ? value : undefined +} diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 4107540..eb6b35a 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -3,7 +3,7 @@ * selection and its filters match. Every filter is defined once, here. The module selects * only; callers enrich with `loadContainerReadViews` (ADR 0009). */ -import { and, eq, getTableColumns, inArray, isNotNull, type SQL } from 'drizzle-orm' +import { and, eq, getTableColumns, inArray, type SQL } from 'drizzle-orm' import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject } from '../../db/schema' import type { ContainerSubtype } from '../container-placement' @@ -11,13 +11,15 @@ import { containerFilterConditions, containerTypeSql, dateRangeConditions, + sourceTypeCondition, specimenFilterConditions, } from './conditions' import { micronixBarcodeScopes, subjectListScopes, type SelectionScope } from './scopes' import type { ContainerSelection, ContainerSelectionResult, SelectedContainer, SelectedSpecimen } from './types' export * from './types' -export { containerHasAllTags } from './conditions' +export { containerHasAllTags, hasContainerLevelFilter } from './conditions' +export { CONTAINER_TYPES, containerTypesFilter, specimenSourceType } from './filters' /** Select the Containers and Specimens a selection matches, with each Container's type. */ export async function selectContainers( @@ -46,16 +48,9 @@ export async function selectContainers( dateTolerance: selection.dateTolerance ?? 0, }) break - case 'database': { - const source = - selection.sourceType === 'subject' - ? isNotNull(specimen.studySubjectId) - : selection.sourceType === 'control' - ? isNotNull(specimen.controlBatchId) - : undefined - scopes = [{ specimens: and(source, ...collectionDate) }] + case 'database': + scopes = [{ specimens: and(sourceTypeCondition(selection.sourceType), ...collectionDate) }] break - } } const specimenConditions = specimenFilterConditions(filters) return runScopes( diff --git a/packages/api/src/lib/container-selection/types.ts b/packages/api/src/lib/container-selection/types.ts index 398be34..30fd9ef 100644 --- a/packages/api/src/lib/container-selection/types.ts +++ b/packages/api/src/lib/container-selection/types.ts @@ -18,8 +18,12 @@ export interface ContainerSelectionFilters { specimenCreated?: DateRange /** When the Container was created. A Container-level filter. */ containerCreated?: DateRange - /** Container types (`ContainerSubtype` values); unknown values match nothing. A Container-level filter. */ - containerTypes?: readonly string[] + /** + * Container types. Omitted means every type; an empty list matches nothing. `'unknown'` + * matches Containers with no subtype row. Narrow wire values with `containerTypesFilter`, + * which drops unrecognised ones. A Container-level filter. + */ + containerTypes?: readonly ContainerSubtype[] /** Containers carrying every one of these Tags. A Container-level filter. */ tagIds?: readonly number[] /** Containers stored in this Location or any Location under it. A Container-level filter. */ @@ -38,6 +42,9 @@ export interface SubjectSelectionEntry { dates?: readonly SubjectDateFilter[] } +/** Where a Specimen comes from: a Subject or a Control batch. */ +export type SpecimenSourceType = 'subject' | 'control' + /** What to select from (ADR 0011). Every id is resolved by the caller; an unknown id matches nothing. */ export type ContainerSelection = | { kind: 'study'; studyId: number; filters?: ContainerSelectionFilters } @@ -52,7 +59,7 @@ export type ContainerSelection = | { kind: 'database' /** Specimens of Subjects or of Control batches; omitted means both. */ - sourceType?: 'subject' | 'control' + sourceType?: SpecimenSourceType filters?: ContainerSelectionFilters } diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index c874e0c..f0adf1c 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,8 +1,7 @@ import { subjectDateSelections, type ContainerExportData, type SubjectDateFilter } from '@sampledb/contract' -import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' -import { study } from '../../db/schema' import { + containerTypesFilter, selectContainers, type ContainerSelection, type ContainerSelectionFilters, @@ -13,7 +12,11 @@ import { enrichContainerData } from './enrich' import { formatAsCSV, formatAsExcel, formatAsJSON } from './format' import { resolveMicronixBarcodesToContainers } from './query' import { buildExportSummary, validateStudyCodes } from './validate' -import { resolveSubjectNamesByStudy, resolveSubjectsByStudyGrouped } from '../identifier-resolution' +import { + resolveStudyRecordByShortCode, + resolveSubjectNamesByStudy, + resolveSubjectsByStudyGrouped, +} from '../identifier-resolution' import type { CSVExportOptions, ExportFilters, @@ -99,13 +102,14 @@ function exportSelectionFilters(filters: Omit): Containe specimenTypeIds: filters.specimen_type_ids, collectionDate: { from: filters.date_from, to: filters.date_to }, containerCreated: { from: filters.created_from, to: filters.created_to }, - containerTypes: filters.container_types, + containerTypes: containerTypesFilter(filters.container_types), tagIds: filters.tag_ids, } } -async function findStudyByShortCode(database: Database, shortCode: string): Promise { - return database.select().from(study).where(eq(study.shortCode, shortCode)).get() +/** Subject names by Subject id, inverting a name to id map. */ +function subjectNamesById(subjectIdsByName: ReadonlyMap): Map { + return new Map([...subjectIdsByName].map(([name, id]) => [id, name])) } async function formatContainerExport( @@ -149,7 +153,7 @@ export async function runSingleStudyContainerExportGet( | { kind: 'json'; json: Awaited>; filename: string } > { const { filters, format, columns, csvOptions, userId, countOnly } = params - const studyRecord = await findStudyByShortCode(database, filters.study) + const studyRecord = await resolveStudyRecordByShortCode(database, filters.study) if (!studyRecord) { throw new ExpectedNotFoundError(`Study with short code '${filters.study}' not found`) } @@ -225,16 +229,13 @@ export async function runSingleStudyContainerExportPost( dateTolerance = 0, } = params - const studyRecord = await findStudyByShortCode(database, studyCode) + const studyRecord = await resolveStudyRecordByShortCode(database, studyCode) if (!studyRecord) { throw new ExpectedNotFoundError('Study not found') } const subjectNameToId = await resolveSubjectNamesByStudy(database, subjectNames, studyRecord.id) - const subjectIdToName = new Map() - for (const [name, id] of subjectNameToId.entries()) { - subjectIdToName.set(id, name) - } + const subjectIdToName = subjectNamesById(subjectNameToId) if (subjectNameToId.size === 0) { const summary: ExportSummary = { @@ -410,7 +411,7 @@ export async function runMultiStudyContainerExportPost( for (const [studyId, studyEntries] of entriesByStudy) { const studyRecord = validation.studies.get(studyId)! const subjectNameToId = subjectsByStudy.get(studyId) ?? new Map() - const subjectIdToName = new Map([...subjectNameToId].map(([name, id]) => [id, name])) + const subjectIdToName = subjectNamesById(subjectNameToId) const studyData = enrichedData.filter((row) => row.study_id === studyId) const studySummary = await buildExportSummary( studyData, @@ -443,7 +444,7 @@ export async function runMultiStudyContainerExportPost( enrichedData, format, { study: 'MULTI' }, - validation.studies.values().next().value ?? MULTI_STUDY_RECORD, + MULTI_STUDY_RECORD, columns, csvOptions, userId diff --git a/packages/api/src/lib/identifier-resolution.ts b/packages/api/src/lib/identifier-resolution.ts index 4deff87..b049c11 100644 --- a/packages/api/src/lib/identifier-resolution.ts +++ b/packages/api/src/lib/identifier-resolution.ts @@ -7,13 +7,15 @@ import type { DatabaseOrTransaction } from './db-types' * Resolve study short code to study ID */ export async function resolveStudyByShortCode(database: Database, shortCode: string): Promise { - const studyRecord = await database - .select({ id: study.id }) - .from(study) - .where(eq(study.shortCode, shortCode)) - .get() - - return studyRecord?.id ?? null + return (await resolveStudyRecordByShortCode(database, shortCode))?.id ?? null +} + +/** Resolve a study short code to the Study record, or undefined when no Study has it. */ +export async function resolveStudyRecordByShortCode( + database: Database, + shortCode: string, +): Promise { + return database.select().from(study).where(eq(study.shortCode, shortCode)).get() } /** diff --git a/packages/api/src/lib/statistics/dashboard-stats.ts b/packages/api/src/lib/statistics/dashboard-stats.ts index 2db5e84..2fa83f5 100644 --- a/packages/api/src/lib/statistics/dashboard-stats.ts +++ b/packages/api/src/lib/statistics/dashboard-stats.ts @@ -1,7 +1,13 @@ -import { eq } from 'drizzle-orm' import type { Database } from '../../db/client' -import { study } from '../../db/schema' -import { selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '../container-selection' +import { + containerTypesFilter, + hasContainerLevelFilter, + selectContainers, + specimenSourceType, + type ContainerSelection, + type ContainerSelectionFilters, +} from '../container-selection' +import { resolveStudyByShortCode } from '../identifier-resolution' import { computeContainerAggregates } from './container-aggregates' import { computeSpecimenAggregates } from './specimen-aggregates' import { computeStorageStatistics } from './storage-aggregates' @@ -46,7 +52,7 @@ function selectionFilters(filters: StatisticsFilters): ContainerSelectionFilters specimenTypeIds: specimenTypeId === undefined ? undefined : [specimenTypeId], collectionDate: { from: filters.collection_date_from, to: filters.collection_date_to }, specimenCreated: { from: filters.created_from, to: filters.created_to }, - containerTypes: filters.container_type ? [filters.container_type] : undefined, + containerTypes: containerTypesFilter(filters.container_type ? [filters.container_type] : undefined), tagIds: filters.tag_ids, locationId: parseId(filters.location_id), } @@ -61,17 +67,12 @@ async function statisticsSelection( database: Database, filters: StatisticsFilters, ): Promise { - const sourceType = - filters.source_type === 'subject' || filters.source_type === 'control' ? filters.source_type : undefined + const sourceType = specimenSourceType(filters.source_type) if (!filters.study) return { kind: 'database', sourceType, filters: selectionFilters(filters) } if (sourceType === 'control') return null - const studyRecord = await database - .select({ id: study.id }) - .from(study) - .where(eq(study.shortCode, filters.study)) - .get() - if (!studyRecord) return null - return { kind: 'study', studyId: studyRecord.id, filters: selectionFilters(filters) } + const studyId = await resolveStudyByShortCode(database, filters.study) + if (studyId === null) return null + return { kind: 'study', studyId, filters: selectionFilters(filters) } } /** Filtered dashboard statistics for specimens, containers, and storage. */ @@ -84,10 +85,8 @@ export async function getDashboardStatistics( const { containers, specimens, containerTypes } = await selectContainers(database, selection) if (specimens.length === 0) return emptyDashboardStatistics() - const hasContainerFilter = - !!filters.container_type || (filters.tag_ids?.length ?? 0) > 0 || parseId(filters.location_id) !== undefined const specimenAggregates = await computeSpecimenAggregates(database, specimens, { - useSpecimenTypeCache: !hasContainerFilter, + useSpecimenTypeCache: !hasContainerLevelFilter(selectionFilters(filters)), }) const containerTotal = containers.length const averagePerSpecimen = specimenAggregates.total > 0 ? containerTotal / specimenAggregates.total : 0 From d0c2b51f60767d16b1b63f8a5a95276f60e877d7 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:18:05 -0700 Subject: [PATCH 13/28] fix(export): select available types through Container selection. GET /export/available-types picked a Study's Specimens and Containers with unchunked IN lists that grew with the Study, and failed once its Subjects outnumbered SQLite's bound-parameter limit. It now calls selectContainers with a study selection and reads the specimen types and containerTypes from the result. The response is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../api/src/routes/__tests__/export.test.ts | 64 ++++++++++++++- packages/api/src/routes/export.ts | 78 ++++--------------- 2 files changed, 77 insertions(+), 65 deletions(-) diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index d94ed31..593e3a6 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -6,7 +6,8 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createExportRoutes } from '../export' -import { errorLogs, specimen as specimenTable } from '../../db/schema' +import { errorLogs, specimen as specimenTable, study as studyTable, unit as unitTable } from '../../db/schema' +import { SQL_IN_CHUNK_SIZE } from '../../lib/sql-helpers' import * as specimensCsv from '../../lib/export/specimens-csv' import { createTestStudy, @@ -458,6 +459,67 @@ describe('Export API', () => { }) }) + describe('GET /api/export/available-types', () => { + it('returns 400 when study parameter is missing', async () => { + const res = await ctx.request('/api/export/available-types', { method: 'GET' }) + expect(res.status).toBe(400) + }) + + it('returns 404 when study does not exist', async () => { + const res = await ctx.request('/api/export/available-types?study=NOPE', { method: 'GET' }) + expect(res.status).toBe(404) + expect(await res.json()).toMatchObject({ error: 'Study not found' }) + }) + + it("lists the Study's specimen types and container types", async () => { + await seedTypedContainers('TYPES', { cryovial: true }) + const other = await createTestStudy(ctx.db, { title: 'Other', shortCode: 'OTHER' }) + const otherSubject = await createTestStudySubject(ctx.db, { studyId: other.id, name: 'Subj1' }) + const plasma = await createTestSpecimenType(ctx.db, { name: 'Plasma' }) + const [unit] = await ctx.db.select().from(unitTable) + await createTestStorageContainer(ctx.db, { + specimenId: (await createTestSpecimen(ctx.db, plasma.id, { studySubjectId: otherSubject.id })).id, + unitId: unit.id, + }) + + const res = await ctx.request('/api/export/available-types?study=TYPES', { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { specimen_types: Array<{ id: number; name: string }>; container_types: string[] } + expect(data.specimen_types.map((type) => type.name)).toEqual(['Blood']) + expect(data.container_types).toEqual(['micronix_tube', 'cryovial_tube']) + }) + + it('returns empty lists for a Study with Specimens but no Containers', async () => { + const study = await createTestStudy(ctx.db, { title: 'Bare', shortCode: 'BARE' }) + const subject = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Subj1' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Plasma' }) + await createTestSpecimen(ctx.db, specimenType.id, { studySubjectId: subject.id }) + + const res = await ctx.request('/api/export/available-types?study=BARE', { method: 'GET' }) + + expect(await res.json()).toEqual({ specimen_types: [{ id: specimenType.id, name: 'Plasma' }], container_types: [] }) + }) + + it('lists the types of a Study with more Subjects than SQLite allows as bound parameters', async () => { + await seedTypedContainers('BIG', { cryovial: false }) + const [big] = await ctx.db.select().from(studyTable).where(eq(studyTable.shortCode, 'BIG')) + ctx.sqlite.exec('BEGIN') + const addSubject = ctx.sqlite.prepare( + "INSERT INTO study_subject (study_id, name, created, last_updated) VALUES (?, ?, '2024-01-01', '2024-01-01')", + ) + for (let i = 0; i < SQL_IN_CHUNK_SIZE * 140; i++) addSubject.run(big.id, `S${i}`) + ctx.sqlite.exec('COMMIT') + + const res = await ctx.request('/api/export/available-types?study=BIG', { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { specimen_types: Array<{ name: string }>; container_types: string[] } + expect(data.specimen_types.map((type) => type.name)).toEqual(['Blood']) + expect(data.container_types).toEqual(['micronix_tube']) + }, 60_000) + }) + describe('error persistence', () => { it('persists export 500 failures to error_logs with stable client body', async () => { const spy = vi diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index 54fc36d..ad9ad8e 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -6,18 +6,10 @@ import { ExpectedNotFoundError, RouteError, } from '../lib/error-handler' -import { - specimen, - studySubject, - study, - specimenType, - storageContainer, - micronixTube, - cryovialTube, - paper, - staticWell, -} from '../db/schema' -import { eq, inArray } from 'drizzle-orm' +import { specimenType } from '../db/schema' +import { inArray } from 'drizzle-orm' +import { CONTAINER_TYPES, selectContainers } from '../lib/container-selection' +import { resolveStudyByShortCode } from '../lib/identifier-resolution' import { formatExportTimestamp, parseCSVExportOptions, @@ -246,70 +238,28 @@ export function createExportRoutes(database: Database): Hono { return c.json({ error: 'Study parameter is required' }, 400) } - const studyRecord = await database.select().from(study).where(eq(study.shortCode, studyCode)).get() - if (!studyRecord) { + const studyId = await resolveStudyByShortCode(database, studyCode) + if (studyId === null) { throw new ExpectedNotFoundError('Study not found') } - const subjects = await database - .select({ id: studySubject.id }) - .from(studySubject) - .where(eq(studySubject.studyId, studyRecord.id)) - - const subjectIds = subjects.map((s) => s.id) - if (subjectIds.length === 0) { - return c.json({ specimen_types: [], container_types: [] }) - } - - let specimensQuery = database - .select({ id: specimen.id, specimenTypeId: specimen.specimenTypeId }) - .from(specimen) - - specimensQuery = - subjectIds.length === 1 - ? (specimensQuery.where(eq(specimen.studySubjectId, subjectIds[0])) as typeof specimensQuery) - : (specimensQuery.where(inArray(specimen.studySubjectId, subjectIds)) as typeof specimensQuery) - - const specimens = await specimensQuery - const specimenIds = specimens.map((s) => s.id) - if (specimenIds.length === 0) { + const selected = await selectContainers(database, { kind: 'study', studyId }) + if (selected.specimens.length === 0) { return c.json({ specimen_types: [], container_types: [] }) } - const specimenTypeIds = [...new Set(specimens.map((s) => s.specimenTypeId))] + const specimenTypeIds = [...new Set(selected.specimens.map((row) => row.specimenTypeId))] const specimenTypes = await database - .select() + .select({ id: specimenType.id, name: specimenType.name }) .from(specimenType) .where(inArray(specimenType.id, specimenTypeIds)) + .orderBy(specimenType.id) - const containers = await database - .select({ id: storageContainer.id }) - .from(storageContainer) - .where(inArray(storageContainer.specimenId, specimenIds)) - - const containerIds = containers.map((row) => row.id) - if (containerIds.length === 0) { - return c.json({ - specimen_types: specimenTypes.map((st) => ({ id: st.id, name: st.name })), - container_types: [], - }) - } - - const [micronixTubes, cryovialTubes, papers, staticWells] = await Promise.all([ - database.select({ id: micronixTube.id }).from(micronixTube).where(inArray(micronixTube.id, containerIds)), - database.select({ id: cryovialTube.id }).from(cryovialTube).where(inArray(cryovialTube.id, containerIds)), - database.select({ id: paper.id }).from(paper).where(inArray(paper.id, containerIds)), - database.select({ id: staticWell.id }).from(staticWell).where(inArray(staticWell.id, containerIds)), - ]) - - const containerTypes: string[] = [] - if (micronixTubes.length > 0) containerTypes.push('micronix_tube') - if (cryovialTubes.length > 0) containerTypes.push('cryovial_tube') - if (papers.length > 0) containerTypes.push('paper') - if (staticWells.length > 0) containerTypes.push('static_well') + const present = new Set(selected.containerTypes.values()) + const containerTypes = CONTAINER_TYPES.filter((type) => type !== 'unknown' && present.has(type)) return c.json({ - specimen_types: specimenTypes.map((st) => ({ id: st.id, name: st.name })), + specimen_types: specimenTypes, container_types: containerTypes, }) } catch (error) { From 31b046a0f5ad886dd941ee1acd410f8a439b68ae Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:18:50 -0700 Subject: [PATCH 14/28] test(api): seed Container selection tests through factories. createTestStorageContainer takes a created override, createTestSpecimen takes collectionDate and created, and createTestContainerTags links Tags to a Container. The Container selection tests use them instead of raw inserts and updates. Co-Authored-By: Claude Opus 5.5 --- .../api/src/__tests__/helpers/factories.ts | 19 +++++-- .../__tests__/container-selection.test.ts | 52 ++++++------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/packages/api/src/__tests__/helpers/factories.ts b/packages/api/src/__tests__/helpers/factories.ts index e5d7feb..78f7c8f 100644 --- a/packages/api/src/__tests__/helpers/factories.ts +++ b/packages/api/src/__tests__/helpers/factories.ts @@ -21,6 +21,7 @@ import { cryovialTube, staticWell, paper, + storageContainerTag, } from '../../db/schema' import { utcNow } from '../../lib/datetime' @@ -103,7 +104,7 @@ export async function createTestUnit(db: Database, data: { */ export async function createTestStorageContainer( db: Database, - options?: { specimenId?: number; unitId?: number } + options?: { specimenId?: number; unitId?: number; created?: string } ) { // Create unit if not provided let unitId = options?.unitId @@ -129,10 +130,21 @@ export async function createTestStorageContainer( unitId: unitId as number, totalQuantity: 1.0, remainingQuantity: 1.0, + ...(options?.created ? { created: options.created } : {}), }).returning() return result } +/** + * Link Tags to a storage container + */ +export async function createTestContainerTags(db: Database, storageContainerId: number, tagIds: readonly number[]) { + return db + .insert(storageContainerTag) + .values(tagIds.map((tagId) => ({ storageContainerId, tagId }))) + .returning() +} + /** * Create a test micronix plate (for collections route tests) */ @@ -348,7 +360,7 @@ export async function createTestStudySubject(db: Database, data: { studyId: numb export async function createTestSpecimen( db: Database, specimenTypeId: number, - options: { studySubjectId?: number; controlBatchId?: number } = {} + options: { studySubjectId?: number; controlBatchId?: number; collectionDate?: string; created?: string } = {} ) { const now = utcNow() let { studySubjectId, controlBatchId } = options @@ -370,7 +382,8 @@ export async function createTestSpecimen( specimenTypeId, studySubjectId, controlBatchId, - created: now, + collectionDate: options.collectionDate, + created: options.created ?? now, lastUpdated: now, }).returning() return result diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index 5fa49e4..ac46b29 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { count, eq } from 'drizzle-orm' +import { count } from 'drizzle-orm' import type { Database as SQLiteDatabase } from 'bun:sqlite' import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' import { SQL_IN_CHUNK_SIZE } from '../../sql-helpers' @@ -7,6 +7,7 @@ import { createTestControlBatch, createTestControlDefinition, createTestCryovialBox, + createTestContainerTags, createTestCryovialTube, createTestLocation, createTestMicronixPlate, @@ -26,11 +27,7 @@ import { createTestUnit, } from '../../../__tests__/helpers/factories' import type { Database } from '../../../db/client' -import { - specimen as specimenTable, - storageContainer as storageContainerTable, - storageContainerTag, -} from '../../../db/schema' +import { storageContainer as storageContainerTable } from '../../../db/schema' import { containerHasAllTags, containerTypesFilter, selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '..' describe('selectContainers', () => { @@ -65,15 +62,11 @@ describe('selectContainers', () => { source: { studySubjectId: number } | { controlBatchId: number }, fields: { collectionDate?: string; created?: string; specimenTypeId?: number } = {}, ) { - const row = await createTestSpecimen(db, fields.specimenTypeId ?? specimenTypeId, source) - await db - .update(specimenTable) - .set({ - collectionDate: fields.collectionDate ?? null, - ...(fields.created ? { created: fields.created } : {}), - }) - .where(eq(specimenTable.id, row.id)) - return row + return createTestSpecimen(db, fields.specimenTypeId ?? specimenTypeId, { + ...source, + collectionDate: fields.collectionDate, + created: fields.created, + }) } type ContainerKind = 'micronix_tube' | 'cryovial_tube' | 'untyped' @@ -82,13 +75,7 @@ describe('selectContainers', () => { specimenId: number, options: { kind?: ContainerKind; created?: string; collectionId?: number; barcode?: string } = {}, ) { - const container = await createTestStorageContainer(db, { specimenId, unitId }) - if (options.created) { - await db - .update(storageContainerTable) - .set({ created: options.created }) - .where(eq(storageContainerTable.id, container.id)) - } + const container = await createTestStorageContainer(db, { specimenId, unitId, created: options.created }) const kind = options.kind ?? 'micronix_tube' if (kind === 'micronix_tube') { barcodeCount += 1 @@ -103,10 +90,6 @@ describe('selectContainers', () => { return container } - async function tag(containerId: number, ...tagIds: number[]) { - await db.insert(storageContainerTag).values(tagIds.map((tagId) => ({ storageContainerId: containerId, tagId }))) - } - const ids = (rows: Array<{ id: number }>) => rows.map((row) => row.id) describe('study selection', () => { @@ -263,9 +246,9 @@ describe('selectContainers', () => { const blue = (await createTestTag(db, { name: 'blue' })).id const both = await seedSpecimen({ studySubjectId: subject.id }) const bothTube = await seedContainer(both.id) - await tag(bothTube.id, red, blue) + await createTestContainerTags(db, bothTube.id, [red, blue]) const redOnly = await seedSpecimen({ studySubjectId: subject.id }) - await tag((await seedContainer(redOnly.id)).id, red) + await createTestContainerTags(db, (await seedContainer(redOnly.id)).id, [red]) const result = await selectContainers(db, select(study.id, subject.id, { tagIds: [red, blue] })) @@ -278,7 +261,7 @@ describe('selectContainers', () => { const red = (await createTestTag(db, { name: 'red' })).id const subject = await seedSubject(study.id, 'P1') const specimen = await seedSpecimen({ studySubjectId: subject.id }) - await tag((await seedContainer(specimen.id)).id, red) + await createTestContainerTags(db, (await seedContainer(specimen.id)).id, [red]) const result = await selectContainers(db, select(study.id, subject.id, { tagIds: [red, red + 999] })) @@ -537,14 +520,9 @@ describe('containerHasAllTags', () => { const specimenId = (await createTestSpecimen(db, specimenTypeId)).id const containers = [] for (let i = 0; i < 4; i++) containers.push(await createTestStorageContainer(db, { specimenId, unitId })) - const links = [ - { storageContainerId: containers[0].id, tagId: red }, - { storageContainerId: containers[0].id, tagId: blue }, - { storageContainerId: containers[1].id, tagId: red }, - { storageContainerId: containers[2].id, tagId: red }, - { storageContainerId: containers[2].id, tagId: blue }, - ] - await db.insert(storageContainerTag).values(links) + await createTestContainerTags(db, containers[0].id, [red, blue]) + await createTestContainerTags(db, containers[1].id, [red]) + await createTestContainerTags(db, containers[2].id, [red, blue]) const where = containerHasAllTags([red, blue, red]) const page = await db From 974d85d424cc7310e6eecd3ac72667ece5b05405 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:21:08 -0700 Subject: [PATCH 15/28] fix(api): bound the parameters of each subject-list query. subjectListScopes chunked by Subject, but each dated Subject added a condition per date to one OR, so 500 Subjects with 80 dates each bound 80,500 parameters and the query failed. Each listed date is now its own condition, and a query takes at most SQL_IN_CHUNK_SIZE of them; undated Subjects get their own query. The list is named subjectDateConditions, since it no longer mixes in the undated condition. Scopes can now overlap, so selectContainers collects Containers by id. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/container-selection.test.ts | 26 +++++++++++++++++++ .../api/src/lib/container-selection/index.ts | 8 +++--- .../api/src/lib/container-selection/scopes.ts | 19 +++++++++++--- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index ac46b29..109d54e 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -430,6 +430,32 @@ describe('selectContainers', () => { expect(ids(result.specimens)).toEqual([first.id, last.id]) }) + + it('selects every listed Subject when known Subjects each list many dates', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subjects = [] + for (let i = 0; i < SQL_IN_CHUNK_SIZE; i++) subjects.push(await seedSubject(study.id, `P${i}`)) + // Days 1 to 80 of 2023, one per visit: far more date conditions than one query can bind. + const dates = Array.from({ length: 80 }, (_, day) => new Date(Date.UTC(2023, 0, day + 1)).toISOString().slice(0, 10)) + const first = await seedSpecimen({ studySubjectId: subjects[0].id }, { collectionDate: '2023-01-01' }) + const firstTube = await seedContainer(first.id) + await seedSpecimen({ studySubjectId: subjects[0].id }, { collectionDate: '2024-01-01' }) + const last = await seedSpecimen({ studySubjectId: subjects.at(-1)!.id }, { collectionDate: '2023-03-21' }) + const lastTube = await seedContainer(last.id) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: subjects.map((subject) => ({ + studyId: study.id, + subjectId: subject.id, + dates: dates.map((exact) => ({ exact })), + })), + dateTolerance: 1, + }) + + expect(ids(result.specimens)).toEqual([first.id, last.id]) + expect(ids(result.containers)).toEqual([firstTube.id, lastTube.id]) + }, 60_000) }) describe('whole-database selection', () => { diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index eb6b35a..2d641fd 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -62,7 +62,8 @@ export async function selectContainers( /** * Run each scope (one per chunk of a listed selection) and collect its Specimens and the - * Containers that also pass the Container-level conditions. With any Container-level + * Containers that also pass the Container-level conditions. Scopes may overlap, so rows are + * collected by id. With any Container-level * condition, only Specimens with a matching Container are kept. */ async function runScopes( @@ -71,7 +72,7 @@ async function runScopes( containerConditions: SQL[], ): Promise { const specimensById = new Map() - const containers: SelectedContainer[] = [] + const containersById = new Map() const containerTypes = new Map() for (const scope of scopes) { for (const row of await database.select().from(specimen).where(scope.specimens)) specimensById.set(row.id, row) @@ -81,10 +82,11 @@ async function runScopes( .innerJoin(specimen, eq(storageContainer.specimenId, specimen.id)) .where(and(scope.specimens, scope.containers, ...containerConditions)) for (const row of rows) { - containers.push(row.container) + containersById.set(row.container.id, row.container) containerTypes.set(row.container.id, row.containerType) } } + const containers = [...containersById.values()] let specimens = [...specimensById.values()] if (containerConditions.length > 0) { const stored = new Set(containers.map((container) => container.specimenId)) diff --git a/packages/api/src/lib/container-selection/scopes.ts b/packages/api/src/lib/container-selection/scopes.ts index 5e554c6..f68b5a4 100644 --- a/packages/api/src/lib/container-selection/scopes.ts +++ b/packages/api/src/lib/container-selection/scopes.ts @@ -17,6 +17,12 @@ export interface SelectionScope { containers?: SQL } +/** + * Per-Subject date conditions per query. Each binds at most three parameters, so a query + * stays under a fixed bound however many dates a Subject lists. + */ +const SUBJECT_DATE_CONDITIONS_PER_QUERY = SQL_IN_CHUNK_SIZE + /** Exactly the micronix tubes with these barcodes, and their Specimens, whatever their Source. */ export function micronixBarcodeScopes(database: Database, barcodes: readonly string[]): SelectionScope[] { const unique = [...new Set(barcodes.filter((barcode) => barcode !== ''))] @@ -61,7 +67,8 @@ export async function subjectListScopes( .from(studySubject) .where(inArray(studySubject.id, subjectIds)) const undated: number[] = [] - const dated: SQL[] = [] + // One condition per listed date of a dated Subject, so the OR grows with dates, not Subjects. + const subjectDateConditions: SQL[] = [] for (const subject of subjects) { const listed = datesBySubject.get(subject.id)?.find((item) => item.studyId === subject.studyId) if (!listed) continue @@ -72,12 +79,16 @@ export async function subjectListScopes( .map((dateFilter) => subjectDateCondition(dateFilter, options.dateTolerance)) .filter((condition) => condition !== undefined) if (dateConditions.length === 0) undated.push(subject.id) - else dated.push(and(eq(specimen.studySubjectId, subject.id), or(...dateConditions))!) + for (const condition of dateConditions) { + subjectDateConditions.push(and(eq(specimen.studySubjectId, subject.id), condition)!) + } } if (undated.length > 0) { - dated.push(and(inArray(specimen.studySubjectId, undated), ...options.collectionDate)!) + scopes.push({ specimens: and(inArray(specimen.studySubjectId, undated), ...options.collectionDate) }) + } + for (const conditions of chunkArray(subjectDateConditions, SUBJECT_DATE_CONDITIONS_PER_QUERY)) { + scopes.push({ specimens: or(...conditions) }) } - if (dated.length > 0) scopes.push({ specimens: or(...dated) }) } return scopes } From 3e79a118da6ce0898c8653d58f8998d3ff939602 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 22 Oct 2026 17:21:48 -0700 Subject: [PATCH 16/28] test(api): cover Container selection filters across selection kinds. The filter table already ran each filter on study, subject-list and whole-database selections. It now also runs them on a whole-database selection narrowed to Subjects, and a subject-list test applies the other filters to a Subject's own dates. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/container-selection.test.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index 109d54e..d949042 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -144,6 +144,7 @@ describe('selectContainers', () => { ['study', (studyId, _subjectId, filters) => ({ kind: 'study', studyId, filters })], ['subject-list', (studyId, subjectId, filters) => ({ kind: 'subjects', subjects: [{ studyId, subjectId }], filters })], ['whole-database', (_studyId, _subjectId, filters) => ({ kind: 'database', filters })], + ['Subject-only whole-database', (_studyId, _subjectId, filters) => ({ kind: 'database', sourceType: 'subject', filters })], ] describe.each(selectionKinds)('filters on a %s selection', (_kind, select) => { @@ -359,6 +360,26 @@ describe('selectContainers', () => { expect(ids(result.specimens)).toEqual([visitOne.id, visitTwo.id, inRange.id]) }) + it("applies the other filters to a Subject's own dates", async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const plasmaTypeId = (await createTestSpecimenType(db, { name: 'Plasma' })).id + const blood = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-03-01' }) + const bloodVial = await seedContainer(blood.id, { kind: 'cryovial_tube' }) + await seedContainer(blood.id, { kind: 'micronix_tube' }) + const plasma = await seedSpecimen({ studySubjectId: subject.id }, { collectionDate: '2024-03-01', specimenTypeId: plasmaTypeId }) + await seedContainer(plasma.id, { kind: 'cryovial_tube' }) + + const result = await selectContainers(db, { + kind: 'subjects', + subjects: [{ studyId: study.id, subjectId: subject.id, dates: [{ exact: '2024-03-01' }] }], + filters: { specimenTypeIds: [specimenTypeId], containerTypes: ['cryovial_tube'] }, + }) + + expect(ids(result.specimens)).toEqual([blood.id]) + expect(ids(result.containers)).toEqual([bloodVial.id]) + }) + it('widens exact per-Subject dates by the tolerance, but not date ranges', async () => { const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) const exact = await seedSubject(study.id, 'P1') From 8eaa527d10e31278116a9d53afd64073439dba76 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:19:50 -0700 Subject: [PATCH 17/28] fix(export): treat a null subject date as undated, and reject malformed subject_dates. A null subject_dates entry reached Container selection as [null] and the export returned 500. Null means undated again, as before the migration, and the route checks the shape of subject_dates, answering 400 when it is wrong. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/export/container-export.ts | 6 ++-- .../api/src/routes/__tests__/export.test.ts | 30 +++++++++++++++++++ packages/api/src/routes/export.ts | 20 +++++++++++-- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index f0adf1c..0adb5f5 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -211,8 +211,8 @@ export async function runSingleStudyContainerExportPost( csvOptions?: CSVExportOptions userId?: number | null countOnly?: boolean - /** Per-Subject collection dates, keyed by subject name. */ - subjectDates?: Record + /** Per-Subject collection dates, keyed by subject name. A null entry means undated. */ + subjectDates?: Record dateTolerance?: number } ): Promise { @@ -265,7 +265,7 @@ export async function runSingleStudyContainerExportPost( return { studyId: studyRecord.id, subjectId, - dates: dates === undefined ? undefined : Array.isArray(dates) ? dates : [dates], + dates: dates == null ? undefined : Array.isArray(dates) ? dates : [dates], } }) const result = await selectContainers(database, { diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 593e3a6..7bf533f 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -389,6 +389,36 @@ describe('Export API', () => { expect(exported.rows.map(([id]) => id).sort((a, b) => a - b)).toEqual([datedJan, undatedFeb]) expect(exported.summary.total_containers).toBe(2) }) + + it('treats a Subject whose subject_dates entry is null as undated', async () => { + const { micronixId } = await seedTypedContainers('PNULL', { cryovial: false }) + + const exported = await postCsvExport({ + study: 'PNULL', + subject_names: ['Subj1'], + subject_dates: { Subj1: null }, + }) + + expect(exported.rows).toEqual([[micronixId, 'micronix_tube']]) + }) + + it.each([ + ['a string', '2024-01-01'], + ['an array', [{ exact: '2024-01-01' }]], + ['an entry that is not a date filter', { Subj1: 'yesterday' }], + ['an exact date that is not a string', { Subj1: { exact: 20240101 } }], + ['a date filter with an unknown key', { Subj1: [{ on: '2024-01-01' }] }], + ])('returns 400 when subject_dates is %s', async (_, subjectDates) => { + await seedTypedContainers('PBAD', { cryovial: false }) + + const res = await ctx.request('/api/export/containers', { + method: 'POST', + json: { study: 'PBAD', subject_names: ['Subj1'], subject_dates: subjectDates, format: 'csv' }, + }) + + expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ errorCode: 'VALIDATION_ERROR' }) + }) }) describe('POST /api/export/containers/validate-studies', () => { diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index ad9ad8e..9fcd9ff 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -8,6 +8,7 @@ import { } from '../lib/error-handler' import { specimenType } from '../db/schema' import { inArray } from 'drizzle-orm' +import { z } from 'zod' import { CONTAINER_TYPES, selectContainers } from '../lib/container-selection' import { resolveStudyByShortCode } from '../lib/identifier-resolution' import { @@ -39,8 +40,22 @@ function parseTagIdsFromBody(body: unknown): number[] | undefined { return ids.length > 0 ? ids : undefined } +/** One per-Subject collection date: an exact date or a range. */ +const subjectDateFilterSchema = z.union([ + z.strictObject({ exact: z.string() }), + z.strictObject({ from: z.string().optional(), to: z.string().optional() }), +]) + +/** + * `subject_dates` of a single-study POST export: per subject name, one date filter, several + * (one per visit), or null for undated. + */ +const subjectDatesSchema = z + .record(z.string(), z.union([subjectDateFilterSchema, z.array(subjectDateFilterSchema), z.null()])) + .nullish() + function handleExportFailure(c: Context, message: string, error: unknown): Response { - if (error instanceof ExpectedNotFoundError || error instanceof RouteError) { + if (error instanceof ExpectedNotFoundError || error instanceof RouteError || error instanceof z.ZodError) { return handleRouteError(error, c) } const details = error instanceof Error ? error.message : String(error) @@ -205,8 +220,7 @@ export function createExportRoutes(database: Database): Hono { return c.json({ error: 'subject_names array is required' }, 400) } - const subjectDates = - body.subject_dates && typeof body.subject_dates === 'object' ? body.subject_dates : undefined + const subjectDates = subjectDatesSchema.parse(body.subject_dates) ?? undefined const result = await runSingleStudyContainerExportPost(database, { studyCode: body.study, From 500f3d50e20fed4ff7278d95761eb528ba3b55b9 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:20:58 -0700 Subject: [PATCH 18/28] fix(export): resolve uploaded barcodes and subject names a chunk at a time. resolveMicronixBarcodesToContainers and resolveSubjectNamesByStudy bound every uploaded value in one IN list, so a long enough upload exceeded SQLite's parameter cap. ADR 0011 keeps queries bounded. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/identifier-resolution.test.ts | 26 ++++++++++++++++ .../src/lib/export/__tests__/query.test.ts | 30 ++++++++++++++++++ packages/api/src/lib/export/query.ts | 22 +++++++------ packages/api/src/lib/identifier-resolution.ts | 31 +++++++------------ 4 files changed, 79 insertions(+), 30 deletions(-) diff --git a/packages/api/src/lib/__tests__/identifier-resolution.test.ts b/packages/api/src/lib/__tests__/identifier-resolution.test.ts index cbf2e27..dfa88e6 100644 --- a/packages/api/src/lib/__tests__/identifier-resolution.test.ts +++ b/packages/api/src/lib/__tests__/identifier-resolution.test.ts @@ -7,7 +7,9 @@ import { resolveContainerByBarcode, resolveStudiesByShortCodes, resolveSpecimenTypesByNames, + resolveSubjectNamesByStudy, } from '../identifier-resolution' +import { SQL_IN_CHUNK_SIZE } from '../sql-helpers' import { createTestStudy, createTestStudySubject, createTestSpecimenType, createTestStorageContainer, createTestStorageType, createTestLocation } from '../../__tests__/helpers/factories' import { box, sheet, paper } from '../../db/schema' import { utcNow } from '../datetime' @@ -132,4 +134,28 @@ describe('identifier-resolution lib', () => { expect(map.get('Serum')).toBe(st.id) }) }) + + describe('resolveSubjectNamesByStudy', () => { + it("resolves the Study's Subjects by name, and not another Study's", async () => { + const study = await createTestStudy(testDb, { title: 'S', shortCode: 'S1' }) + const other = await createTestStudy(testDb, { title: 'O', shortCode: 'O1' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'P1' }) + await createTestStudySubject(testDb, { studyId: other.id, name: 'P2' }) + + const map = await resolveSubjectNamesByStudy(testDb, ['P1', 'P2', 'P1'], study.id) + + expect(map).toEqual(new Map([['P1', subject.id]])) + }) + + it('resolves every listed name when the list has more names than SQLite allows as bound parameters', async () => { + const study = await createTestStudy(testDb, { title: 'S', shortCode: 'S1' }) + const first = await createTestStudySubject(testDb, { studyId: study.id, name: 'FIRST' }) + const last = await createTestStudySubject(testDb, { studyId: study.id, name: 'LAST' }) + const unknown = Array.from({ length: SQL_IN_CHUNK_SIZE * 140 }, (_, i) => `UNKNOWN${i}`) + + const map = await resolveSubjectNamesByStudy(testDb, ['FIRST', ...unknown, 'LAST'], study.id) + + expect(map).toEqual(new Map([['FIRST', first.id], ['LAST', last.id]])) + }) + }) }) diff --git a/packages/api/src/lib/export/__tests__/query.test.ts b/packages/api/src/lib/export/__tests__/query.test.ts index 3578002..f6b98a7 100644 --- a/packages/api/src/lib/export/__tests__/query.test.ts +++ b/packages/api/src/lib/export/__tests__/query.test.ts @@ -1,6 +1,15 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' import type { Database } from '../../../db/client' +import { SQL_IN_CHUNK_SIZE } from '../../sql-helpers' +import { + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestStorageContainer, + createTestStorageType, + createTestUnit, +} from '../../../__tests__/helpers/factories' import { resolveMicronixBarcodesToContainers } from '../query' describe('resolveMicronixBarcodesToContainers', () => { @@ -31,4 +40,25 @@ describe('resolveMicronixBarcodesToContainers', () => { const result = await resolveMicronixBarcodesToContainers(testDb, [' ', '']) expect(result.size).toBe(0) }) + + it('resolves every listed barcode when the list has more barcodes than SQLite allows as bound parameters', async () => { + const storageType = await createTestStorageType(testDb, { name: 'Freezer -80' }) + const freezer = await createTestLocation(testDb, { + name: 'Freezer', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate', locationId: freezer.id }) + const unitId = (await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' })).id as number + const specimenId = (await createTestStorageContainer(testDb, { unitId })).specimenId + const first = await createTestStorageContainer(testDb, { unitId, specimenId }) + await createTestMicronixTube(testDb, { containerId: first.id, collectionId: plate.id, barcode: 'FIRST' }) + const last = await createTestStorageContainer(testDb, { unitId, specimenId }) + await createTestMicronixTube(testDb, { containerId: last.id, collectionId: plate.id, barcode: 'LAST' }) + const unknown = Array.from({ length: SQL_IN_CHUNK_SIZE * 140 }, (_, i) => `UNKNOWN${i}`) + + const result = await resolveMicronixBarcodesToContainers(testDb, ['FIRST', ...unknown, 'LAST']) + + expect(result).toEqual(new Map([['FIRST', first.id], ['LAST', last.id]])) + }) }) diff --git a/packages/api/src/lib/export/query.ts b/packages/api/src/lib/export/query.ts index 50c755a..df4dbcf 100644 --- a/packages/api/src/lib/export/query.ts +++ b/packages/api/src/lib/export/query.ts @@ -1,10 +1,11 @@ import { inArray } from 'drizzle-orm' import type { Database } from '../../db/client' import { micronixTube } from '../../db/schema' +import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' /** - * Resolve micronix barcodes to container IDs - * Only queries micronix_tube table (barcodes are globally unique) + * Resolve micronix barcodes to container IDs, a chunk at a time so no IN list grows with + * the upload. Only queries micronix_tube table (barcodes are globally unique) */ export async function resolveMicronixBarcodesToContainers( database: Database, @@ -13,15 +14,16 @@ export async function resolveMicronixBarcodesToContainers( const uniqueBarcodes = [...new Set(barcodes.filter(b => b && b.trim()))] if (uniqueBarcodes.length === 0) return new Map() - const micronixTubes = await database - .select({ id: micronixTube.id, barcode: micronixTube.barcode }) - .from(micronixTube) - .where(inArray(micronixTube.barcode, uniqueBarcodes)) - const result = new Map() - for (const tube of micronixTubes) { - if (tube.barcode) { - result.set(tube.barcode, tube.id) + for (const chunk of chunkArray(uniqueBarcodes, SQL_IN_CHUNK_SIZE)) { + const micronixTubes = await database + .select({ id: micronixTube.id, barcode: micronixTube.barcode }) + .from(micronixTube) + .where(inArray(micronixTube.barcode, chunk)) + for (const tube of micronixTubes) { + if (tube.barcode) { + result.set(tube.barcode, tube.id) + } } } diff --git a/packages/api/src/lib/identifier-resolution.ts b/packages/api/src/lib/identifier-resolution.ts index b049c11..35cc258 100644 --- a/packages/api/src/lib/identifier-resolution.ts +++ b/packages/api/src/lib/identifier-resolution.ts @@ -2,6 +2,7 @@ import type { Database } from '../db/client' import { study, studySubject, specimenType, micronixTube, storageContainer, cryovialTube, paper } from '../db/schema' import { eq, and, inArray } from 'drizzle-orm' import type { DatabaseOrTransaction } from './db-types' +import { chunkArray, SQL_IN_CHUNK_SIZE } from './sql-helpers' /** * Resolve study short code to study ID @@ -140,34 +141,24 @@ export async function resolveSubjectsByNameAndStudy( } /** - * Batch resolve subject names to IDs for a single study (optimized) + * Batch resolve subject names to IDs for a single study, a chunk of names at a time so no + * IN list grows with the upload. */ export async function resolveSubjectNamesByStudy( database: Database, subjectNames: string[], studyId: number ): Promise> { - const uniqueNames = [...new Set(subjectNames)] - if (uniqueNames.length === 0) return new Map() - - const subjects = await database - .select({ - id: studySubject.id, - name: studySubject.name, - }) - .from(studySubject) - .where( - and( - eq(studySubject.studyId, studyId), - inArray(studySubject.name, uniqueNames) - ) as any - ) - const result = new Map() - for (const subject of subjects) { - result.set(subject.name, subject.id) + for (const names of chunkArray([...new Set(subjectNames)], SQL_IN_CHUNK_SIZE)) { + const subjects = await database + .select({ id: studySubject.id, name: studySubject.name }) + .from(studySubject) + .where(and(eq(studySubject.studyId, studyId), inArray(studySubject.name, names))) + for (const subject of subjects) { + result.set(subject.name, subject.id) + } } - return result } From b92b7168179eca97a6eca9856bf9749eba0dbb56 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:21:31 -0700 Subject: [PATCH 19/28] fix(export): report a missing study on available-types with a typed error. GET /export/available-types returned its 400 with c.json, which CODING_STANDARDS.md rules out in changed route code (ADR 0004). It throws ValidationError now; the body keeps its error message and gains errorCode. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/routes/__tests__/export.test.ts | 1 + packages/api/src/routes/export.ts | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 7bf533f..50ae590 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -493,6 +493,7 @@ describe('Export API', () => { it('returns 400 when study parameter is missing', async () => { const res = await ctx.request('/api/export/available-types', { method: 'GET' }) expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ error: 'Study parameter is required', errorCode: 'VALIDATION_ERROR' }) }) it('returns 404 when study does not exist', async () => { diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index 9fcd9ff..6f3617d 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -5,6 +5,7 @@ import { handleRouteError, ExpectedNotFoundError, RouteError, + ValidationError, } from '../lib/error-handler' import { specimenType } from '../db/schema' import { inArray } from 'drizzle-orm' @@ -55,7 +56,12 @@ const subjectDatesSchema = z .nullish() function handleExportFailure(c: Context, message: string, error: unknown): Response { - if (error instanceof ExpectedNotFoundError || error instanceof RouteError || error instanceof z.ZodError) { + if ( + error instanceof ExpectedNotFoundError || + error instanceof RouteError || + error instanceof ValidationError || + error instanceof z.ZodError + ) { return handleRouteError(error, c) } const details = error instanceof Error ? error.message : String(error) @@ -249,7 +255,7 @@ export function createExportRoutes(database: Database): Hono { try { const studyCode = c.req.query('study') if (!studyCode) { - return c.json({ error: 'Study parameter is required' }, 400) + throw new ValidationError('Study parameter is required') } const studyId = await resolveStudyByShortCode(database, studyCode) From 84b4a6227844f5d2c350f74d4e02cd8e68a01784 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:22:08 -0700 Subject: [PATCH 20/28] test(api): name a Container with no subtype row 'unknown', as the module does. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/container-selection.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index d949042..67df4d5 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -69,7 +69,7 @@ describe('selectContainers', () => { }) } - type ContainerKind = 'micronix_tube' | 'cryovial_tube' | 'untyped' + type ContainerKind = 'micronix_tube' | 'cryovial_tube' | 'unknown' async function seedContainer( specimenId: number, @@ -218,7 +218,7 @@ describe('selectContainers', () => { const tubeOnly = await seedSpecimen({ studySubjectId: subject.id }) await seedContainer(tubeOnly.id, { kind: 'micronix_tube' }) const untypedOnly = await seedSpecimen({ studySubjectId: subject.id }) - const untyped = await seedContainer(untypedOnly.id, { kind: 'untyped' }) + const untyped = await seedContainer(untypedOnly.id, { kind: 'unknown' }) const vials = await selectContainers(db, select(study.id, subject.id, { containerTypes: containerTypesFilter(['cryovial_tube', 'not_a_type']) })) const unknown = await selectContainers(db, select(study.id, subject.id, { containerTypes: ['unknown'] })) @@ -287,14 +287,14 @@ describe('selectContainers', () => { const specimen = await seedSpecimen({ studySubjectId: subject.id }) const tube = await seedContainer(specimen.id, { collectionId: shelfPlate }) const vial = await seedContainer(specimen.id, { kind: 'cryovial_tube', collectionId: rackBox }) - const well = await seedContainer(specimen.id, { kind: 'untyped' }) + const well = await seedContainer(specimen.id, { kind: 'unknown' }) await createTestStaticWell(db, { containerId: well.id, collectionId: shelfPlate }) - const boxedPaper = await seedContainer(specimen.id, { kind: 'untyped' }) + const boxedPaper = await seedContainer(specimen.id, { kind: 'unknown' }) await createTestPaper(db, { containerId: boxedPaper.id, sheetId: shelfSheet }) - const baggedPaper = await seedContainer(specimen.id, { kind: 'untyped' }) + const baggedPaper = await seedContainer(specimen.id, { kind: 'unknown' }) await createTestPaper(db, { containerId: baggedPaper.id, sheetId: rackSheet }) await seedContainer(specimen.id) - await seedContainer(specimen.id, { kind: 'untyped' }) + await seedContainer(specimen.id, { kind: 'unknown' }) const elsewhere = await seedSpecimen({ studySubjectId: subject.id }) await seedContainer(elsewhere.id) From bf8c83d0f490f41b9c9a3d442c6bbfea6d7a3a69 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:22:51 -0700 Subject: [PATCH 21/28] fix(export): read only distinct types for available-types. GET /export/available-types loaded every Specimen and Container row of the Study to list their types. selectContainerSelectionTypes runs the same selection with SELECT DISTINCT and returns only the type lists. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0011-container-selection.md | 1 + .../__tests__/container-selection.test.ts | 44 ++++++++++++- .../api/src/lib/container-selection/index.ts | 65 +++++++++++++++++-- .../api/src/lib/container-selection/types.ts | 8 +++ packages/api/src/routes/export.ts | 27 ++++---- 5 files changed, 122 insertions(+), 23 deletions(-) diff --git a/docs/adr/0011-container-selection.md b/docs/adr/0011-container-selection.md index b081b5d..6f41cc2 100644 --- a/docs/adr/0011-container-selection.md +++ b/docs/adr/0011-container-selection.md @@ -11,6 +11,7 @@ We put **Container selection** (see [CONTEXT.md](../../CONTEXT.md)) behind one m ## Interface - One function takes a selection and returns the matching Container rows, the matching Specimen rows, and each Container's type. It never enriches: callers pass the rows to `loadContainerReadViews` ([ADR 0009](./0009-container-read-view.md)). +- A second function takes the same selection and returns only the distinct specimen types and container types it matches, without loading rows, for callers that list them, such as export's available types. - A selection is one of: - a **Study**; - a subject list of (Study, Subject ids, per-subject dates) entries spanning any number of Studies; diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index 67df4d5..6a101d0 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -28,7 +28,14 @@ import { } from '../../../__tests__/helpers/factories' import type { Database } from '../../../db/client' import { storageContainer as storageContainerTable } from '../../../db/schema' -import { containerHasAllTags, containerTypesFilter, selectContainers, type ContainerSelection, type ContainerSelectionFilters } from '..' +import { + containerHasAllTags, + containerTypesFilter, + selectContainers, + selectContainerSelectionTypes, + type ContainerSelection, + type ContainerSelectionFilters, +} from '..' describe('selectContainers', () => { let db: Database @@ -545,6 +552,41 @@ describe('selectContainers', () => { expect(ids(result.specimens)).toEqual([specimen.id]) }) }) + describe('selectContainerSelectionTypes', () => { + it('lists the distinct specimen types of the matching Specimens and the types of their Containers', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const other = await createTestStudy(db, { title: 'Other', shortCode: 'S2' }) + const subject = await seedSubject(study.id, 'P1') + const plasma = (await createTestSpecimenType(db, { name: 'Plasma' })).id + const serum = (await createTestSpecimenType(db, { name: 'Serum' })).id + const blood = await seedSpecimen({ studySubjectId: subject.id }) + await seedContainer(blood.id, { kind: 'cryovial_tube' }) + await seedContainer(blood.id, { kind: 'cryovial_tube' }) + await seedContainer(blood.id, { kind: 'unknown' }) + await seedSpecimen({ studySubjectId: subject.id }, { specimenTypeId: plasma }) + await seedContainer((await seedSpecimen({ studySubjectId: (await seedSubject(other.id, 'Q1')).id }, { specimenTypeId: serum })).id) + + const types = await selectContainerSelectionTypes(db, { kind: 'study', studyId: study.id }) + + expect(types).toEqual({ specimenTypeIds: [specimenTypeId, plasma], containerTypes: ['cryovial_tube', 'unknown'] }) + }) + + it('lists only the specimen types of Specimens with a matching Container under a Container-level filter', async () => { + const study = await createTestStudy(db, { title: 'Study', shortCode: 'S1' }) + const subject = await seedSubject(study.id, 'P1') + const plasma = (await createTestSpecimenType(db, { name: 'Plasma' })).id + await seedContainer((await seedSpecimen({ studySubjectId: subject.id })).id, { kind: 'cryovial_tube' }) + await seedContainer((await seedSpecimen({ studySubjectId: subject.id }, { specimenTypeId: plasma })).id) + + const types = await selectContainerSelectionTypes(db, { + kind: 'study', + studyId: study.id, + filters: { containerTypes: ['micronix_tube'] }, + }) + + expect(types).toEqual({ specimenTypeIds: [plasma], containerTypes: ['micronix_tube'] }) + }) + }) }) describe('containerHasAllTags', () => { diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 2d641fd..06c46fc 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -15,7 +15,14 @@ import { specimenFilterConditions, } from './conditions' import { micronixBarcodeScopes, subjectListScopes, type SelectionScope } from './scopes' -import type { ContainerSelection, ContainerSelectionResult, SelectedContainer, SelectedSpecimen } from './types' +import { CONTAINER_TYPES } from './filters' +import type { + ContainerSelection, + ContainerSelectionResult, + ContainerSelectionTypes, + SelectedContainer, + SelectedSpecimen, +} from './types' export * from './types' export { containerHasAllTags, hasContainerLevelFilter } from './conditions' @@ -26,8 +33,53 @@ export async function selectContainers( database: Database, selection: ContainerSelection, ): Promise { + const { scopes, containerConditions } = await selectionScopes(database, selection) + return runScopes(database, scopes, containerConditions) +} + +/** + * The specimen types and container types a selection matches, without loading its rows. + * Specimen types follow the same rule as `selectContainers`' Specimens: with a Container-level + * filter, only Specimens with a matching Container count. + */ +export async function selectContainerSelectionTypes( + database: Database, + selection: ContainerSelection, +): Promise { + const { scopes, containerConditions } = await selectionScopes(database, selection) + const specimenTypeIds = new Set() + const containerTypes = new Set() + for (const scope of scopes) { + const rows = await database + .selectDistinct({ specimenTypeId: specimen.specimenTypeId, containerType: containerTypeSql }) + .from(storageContainer) + .innerJoin(specimen, eq(storageContainer.specimenId, specimen.id)) + .where(and(scope.specimens, scope.containers, ...containerConditions)) + for (const row of rows) { + containerTypes.add(row.containerType) + if (containerConditions.length > 0) specimenTypeIds.add(row.specimenTypeId) + } + if (containerConditions.length === 0) { + const specimenRows = await database + .selectDistinct({ specimenTypeId: specimen.specimenTypeId }) + .from(specimen) + .where(scope.specimens) + for (const row of specimenRows) specimenTypeIds.add(row.specimenTypeId) + } + } + return { + specimenTypeIds: [...specimenTypeIds].sort((a, b) => a - b), + containerTypes: CONTAINER_TYPES.filter((type) => containerTypes.has(type)), + } +} + +/** The bounded scopes a selection runs, and the Container-level conditions every scope shares. */ +async function selectionScopes( + database: Database, + selection: ContainerSelection, +): Promise<{ scopes: SelectionScope[]; containerConditions: SQL[] }> { if (selection.kind === 'micronixBarcodes') { - return runScopes(database, micronixBarcodeScopes(database, selection.barcodes), []) + return { scopes: micronixBarcodeScopes(database, selection.barcodes), containerConditions: [] } } const filters = selection.filters ?? {} @@ -53,11 +105,10 @@ export async function selectContainers( break } const specimenConditions = specimenFilterConditions(filters) - return runScopes( - database, - scopes.map((scope) => ({ ...scope, specimens: and(scope.specimens, ...specimenConditions) })), - containerFilterConditions(filters), - ) + return { + scopes: scopes.map((scope) => ({ ...scope, specimens: and(scope.specimens, ...specimenConditions) })), + containerConditions: containerFilterConditions(filters), + } } /** diff --git a/packages/api/src/lib/container-selection/types.ts b/packages/api/src/lib/container-selection/types.ts index 30fd9ef..d63c48b 100644 --- a/packages/api/src/lib/container-selection/types.ts +++ b/packages/api/src/lib/container-selection/types.ts @@ -77,3 +77,11 @@ export interface ContainerSelectionResult { /** Each selected Container's type, by Container id. */ containerTypes: Map } + +/** The specimen types and container types a selection matches. */ +export interface ContainerSelectionTypes { + /** Specimen type ids, ascending. */ + specimenTypeIds: number[] + /** Container types, in `CONTAINER_TYPES` order; `'unknown'` when a Container has no subtype row. */ + containerTypes: ContainerSubtype[] +} diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index 6f3617d..f16808c 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -10,7 +10,7 @@ import { import { specimenType } from '../db/schema' import { inArray } from 'drizzle-orm' import { z } from 'zod' -import { CONTAINER_TYPES, selectContainers } from '../lib/container-selection' +import { selectContainerSelectionTypes } from '../lib/container-selection' import { resolveStudyByShortCode } from '../lib/identifier-resolution' import { formatExportTimestamp, @@ -263,20 +263,17 @@ export function createExportRoutes(database: Database): Hono { throw new ExpectedNotFoundError('Study not found') } - const selected = await selectContainers(database, { kind: 'study', studyId }) - if (selected.specimens.length === 0) { - return c.json({ specimen_types: [], container_types: [] }) - } - - const specimenTypeIds = [...new Set(selected.specimens.map((row) => row.specimenTypeId))] - const specimenTypes = await database - .select({ id: specimenType.id, name: specimenType.name }) - .from(specimenType) - .where(inArray(specimenType.id, specimenTypeIds)) - .orderBy(specimenType.id) - - const present = new Set(selected.containerTypes.values()) - const containerTypes = CONTAINER_TYPES.filter((type) => type !== 'unknown' && present.has(type)) + const types = await selectContainerSelectionTypes(database, { kind: 'study', studyId }) + // Specimen types are a small reference table, so this IN list stays bounded. + const specimenTypes = + types.specimenTypeIds.length === 0 + ? [] + : await database + .select({ id: specimenType.id, name: specimenType.name }) + .from(specimenType) + .where(inArray(specimenType.id, types.specimenTypeIds)) + .orderBy(specimenType.id) + const containerTypes = types.containerTypes.filter((type) => type !== 'unknown') return c.json({ specimen_types: specimenTypes, From a463c69c0c817eff518d43a346f719dbf41cfc7b Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:22:58 -0700 Subject: [PATCH 22/28] docs: state in ADR 0011 that an undated entry wins for a Subject listed twice. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0011-container-selection.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/0011-container-selection.md b/docs/adr/0011-container-selection.md index 6f41cc2..4b1b264 100644 --- a/docs/adr/0011-container-selection.md +++ b/docs/adr/0011-container-selection.md @@ -35,7 +35,7 @@ We put **Container selection** (see [CONTEXT.md](../../CONTEXT.md)) behind one m - **Created**: Specimen created and Container created are separate filters. The routes keep their wire names: export's `created_from`/`created_to` map to Container created, and statistics' to Specimen created. - **Container type**: applied once, here. The GET export's "No containers found" 404 now fires on the filtered result, so a Study whose Containers are all of other types returns 404 instead of an empty file. The type `unknown` matches Containers with no subtype row; any other unrecognised type matches nothing. - **Location**: the Location and every Location under it. -- **Collection dates**: a listed Subject with its own dates ignores the global collection-date range. Undated Subjects get the global range. Tolerance widens exact dates only. +- **Collection dates**: a listed Subject with its own dates ignores the global collection-date range. Undated Subjects get the global range. When a Subject is listed more than once, an undated entry wins: the Subject is undated. Tolerance widens exact dates only. - **Specimens**: with no Container-level filter (container type, Tag, Location, Container created), every matching Specimen is included, with or without Containers. With one, only Specimens that have a matching Container are included. Export ignores Specimens with no Container. ## Considered options From 998b091a950fae1fa12e7ddcd089d72ef56d7a3e Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:23:54 -0700 Subject: [PATCH 23/28] refactor(export): fold a Subject's collection dates in one place. The contract's addSubjectDates now reads one listing of a Subject's dates (a filter, a list, or none) and folds it into the Subject's selection, with an undated listing winning. subjectDateSelections, the subject-list scopes and the single-study POST export use it instead of their own copies. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/container-selection/scopes.ts | 16 +++++------ .../api/src/lib/export/container-export.ts | 28 ++++++++++++------- .../src/__tests__/export-filter-parse.test.ts | 27 ++++++++++++++++++ packages/contract/src/export-filter-parse.ts | 28 +++++++++++++------ packages/contract/src/index.ts | 1 + 5 files changed, 72 insertions(+), 28 deletions(-) diff --git a/packages/api/src/lib/container-selection/scopes.ts b/packages/api/src/lib/container-selection/scopes.ts index f68b5a4..44fdea1 100644 --- a/packages/api/src/lib/container-selection/scopes.ts +++ b/packages/api/src/lib/container-selection/scopes.ts @@ -1,5 +1,5 @@ import { and, eq, inArray, or, type SQL } from 'drizzle-orm' -import type { SubjectDateFilter } from '@sampledb/contract' +import { addSubjectDates, type SubjectDateFilter, type SubjectDateSelection } from '@sampledb/contract' import type { Database } from '../../db/client' import { micronixTube, specimen, storageContainer, studySubject } from '../../db/schema' import { shiftCalendarDate } from '../datetime' @@ -39,25 +39,23 @@ export function micronixBarcodeScopes(database: Database, barcodes: readonly str /** * The listed Subjects, a chunk at a time. A Subject listed under a Study it isn't in matches * nothing. A Subject with its own dates ignores the global collection-date range; one listed - * undated, on any entry, gets the global range. + * undated, on any entry, gets the global range (`addSubjectDates`). */ export async function subjectListScopes( database: Database, entries: readonly SubjectSelectionEntry[], options: { collectionDate: SQL[]; dateTolerance: number }, ): Promise { - const datesBySubject = new Map() + const datesBySubject = new Map() for (const entry of entries) { const listed = datesBySubject.get(entry.subjectId) ?? [] let forStudy = listed.find((item) => item.studyId === entry.studyId) if (!forStudy) { - forStudy = { studyId: entry.studyId, dates: [] } + forStudy = { studyId: entry.studyId, dates: undefined } listed.push(forStudy) datesBySubject.set(entry.subjectId, listed) } - const dateConditions = entry.dates ?? [] - if (dateConditions.length === 0) forStudy.dates = 'all' - else if (forStudy.dates !== 'all') forStudy.dates.push(...dateConditions) + forStudy.dates = addSubjectDates(forStudy.dates, entry.dates) } const scopes: SelectionScope[] = [] @@ -73,9 +71,9 @@ export async function subjectListScopes( const listed = datesBySubject.get(subject.id)?.find((item) => item.studyId === subject.studyId) if (!listed) continue const dateConditions = - listed.dates === 'all' + listed.dates?.kind !== 'dates' ? [] - : listed.dates + : listed.dates.filters .map((dateFilter) => subjectDateCondition(dateFilter, options.dateTolerance)) .filter((condition) => condition !== undefined) if (dateConditions.length === 0) undated.push(subject.id) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index 0adb5f5..e4f1823 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -1,4 +1,10 @@ -import { subjectDateSelections, type ContainerExportData, type SubjectDateFilter } from '@sampledb/contract' +import { + addSubjectDates, + subjectDateSelections, + type ContainerExportData, + type SubjectDateFilter, + type SubjectDateSelection, +} from '@sampledb/contract' import type { Database } from '../../db/client' import { containerTypesFilter, @@ -107,6 +113,11 @@ function exportSelectionFilters(filters: Omit): Containe } } +/** A Subject's dates for a subject-list selection; undefined when it selects all of them. */ +function subjectDateFilters(selection: SubjectDateSelection): SubjectDateFilter[] | undefined { + return selection.kind === 'dates' ? selection.filters : undefined +} + /** Subject names by Subject id, inverting a name to id map. */ function subjectNamesById(subjectIdsByName: ReadonlyMap): Map { return new Map([...subjectIdsByName].map(([name, id]) => [id, name])) @@ -260,14 +271,11 @@ export async function runSingleStudyContainerExportPost( }) } - const subjects: SubjectSelectionEntry[] = [...subjectNameToId].map(([name, subjectId]) => { - const dates = subjectDates?.[name] - return { - studyId: studyRecord.id, - subjectId, - dates: dates == null ? undefined : Array.isArray(dates) ? dates : [dates], - } - }) + const subjects: SubjectSelectionEntry[] = [...subjectNameToId].map(([name, subjectId]) => ({ + studyId: studyRecord.id, + subjectId, + dates: subjectDateFilters(addSubjectDates(undefined, subjectDates?.[name])), + })) const result = await selectContainers(database, { kind: 'subjects', subjects, @@ -391,7 +399,7 @@ export async function runMultiStudyContainerExportPost( for (const [subjectName, selection] of subjectDateSelections(studyEntries)) { const subjectId = subjectIds.get(subjectName) if (subjectId === undefined) continue - subjects.push({ studyId, subjectId, dates: selection.kind === 'dates' ? selection.filters : undefined }) + subjects.push({ studyId, subjectId, dates: subjectDateFilters(selection) }) } } diff --git a/packages/contract/src/__tests__/export-filter-parse.test.ts b/packages/contract/src/__tests__/export-filter-parse.test.ts index f75dfd2..1fd5928 100644 --- a/packages/contract/src/__tests__/export-filter-parse.test.ts +++ b/packages/contract/src/__tests__/export-filter-parse.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'bun:test' import { + addSubjectDates, parseBarcodeExportFilterCsv, parseMultiStudyExportFilterCsv, parseSingleStudyExportFilterCsv, @@ -62,6 +63,32 @@ describe('parseBarcodeExportFilterCsv', () => { }) }) +describe('addSubjectDates', () => { + it('reads a single date filter, a list, or no dates', () => { + expect(addSubjectDates(undefined, { exact: '2024-01-01' })).toEqual({ kind: 'dates', filters: [{ exact: '2024-01-01' }] }) + expect(addSubjectDates(undefined, [{ exact: '2024-01-01' }, { from: '2024-02-01' }])).toEqual({ + kind: 'dates', + filters: [{ exact: '2024-01-01' }, { from: '2024-02-01' }], + }) + expect(addSubjectDates(undefined, null)).toEqual({ kind: 'all' }) + expect(addSubjectDates(undefined, undefined)).toEqual({ kind: 'all' }) + expect(addSubjectDates(undefined, [])).toEqual({ kind: 'all' }) + }) + + it('keeps the dates of every dated listing as alternatives', () => { + const first = addSubjectDates(undefined, { exact: '2024-01-01' }) + expect(addSubjectDates(first, [{ exact: '2024-03-01' }])).toEqual({ + kind: 'dates', + filters: [{ exact: '2024-01-01' }, { exact: '2024-03-01' }], + }) + }) + + it('lets an undated listing win, in either order', () => { + expect(addSubjectDates(addSubjectDates(undefined, { exact: '2024-01-01' }), null)).toEqual({ kind: 'all' }) + expect(addSubjectDates(addSubjectDates(undefined, null), { exact: '2024-01-01' })).toEqual({ kind: 'all' }) + }) +}) + describe('subjectDateSelections', () => { it('keeps every date for a subject listed once per visit', () => { const selections = subjectDateSelections([ diff --git a/packages/contract/src/export-filter-parse.ts b/packages/contract/src/export-filter-parse.ts index 3035f10..2d8cfd9 100644 --- a/packages/contract/src/export-filter-parse.ts +++ b/packages/contract/src/export-filter-parse.ts @@ -202,24 +202,34 @@ export function parseBarcodeExportFilterCsv(csvText: string): string[] { } /** - * Fold subject-list filter rows into one collection-date selection per subject name. - * A subject on several rows keeps every date (one per visit). A row with no date selects - * all of that subject's dates, and wins over the subject's dated rows. + * Add one listing of a Subject's collection dates to the Subject's selection so far. A + * listing is one date filter, several (one per visit), or none: null, undefined or an empty + * list. Dated listings add up as alternatives. A listing with no dates selects all of the + * Subject's dates and wins over its dated listings, in either order. + */ +export function addSubjectDates( + selection: SubjectDateSelection | undefined, + dates: SubjectDateFilter | readonly SubjectDateFilter[] | null | undefined, +): SubjectDateSelection { + if (selection?.kind === 'all') return selection + const listed = dates == null ? [] : Array.isArray(dates) ? dates : [dates as SubjectDateFilter] + if (listed.length === 0) return { kind: 'all' } + return { kind: 'dates', filters: [...(selection?.filters ?? []), ...listed] } +} + +/** + * Fold subject-list filter rows into one collection-date selection per subject name, by + * `addSubjectDates`. A row with no date selects all of that subject's dates. */ export function subjectDateSelections(rows: SingleStudyExportFilterRow[]): Map { const selections = new Map() for (const row of rows) { - const existing = selections.get(row.subject_name) - if (existing?.kind === 'all') continue const dateFilter: SubjectDateFilter | null = row.collection_date ? { exact: row.collection_date } : row.date_from || row.date_to ? { from: row.date_from, to: row.date_to } : null - selections.set( - row.subject_name, - dateFilter ? { kind: 'dates', filters: [...(existing?.filters ?? []), dateFilter] } : { kind: 'all' }, - ) + selections.set(row.subject_name, addSubjectDates(selections.get(row.subject_name), dateFilter)) } return selections } diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1bf79f0..1a9f103 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -24,6 +24,7 @@ export { type ExportColumnKind, } from './export-columns' export { + addSubjectDates, parseBarcodeExportFilterCsv, parseMultiStudyExportFilterCsv, parseSingleStudyExportFilterCsv, From 3a66e65ef3e00163df68048588be929984e1ccc0 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:24:58 -0700 Subject: [PATCH 24/28] refactor(export): format every Container export through one dispatch. The GET export repeated the json/csv/xlsx dispatch of formatContainerExport, and three callers passed the MULTI filters and study record pair by hand. formatContainerExport returns the formatted result for both, and defaults its JSON metadata to the multi-study placeholder. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/export/container-export.ts | 167 ++++++++---------- 1 file changed, 77 insertions(+), 90 deletions(-) diff --git a/packages/api/src/lib/export/container-export.ts b/packages/api/src/lib/export/container-export.ts index e4f1823..96811d0 100644 --- a/packages/api/src/lib/export/container-export.ts +++ b/packages/api/src/lib/export/container-export.ts @@ -123,28 +123,71 @@ function subjectNamesById(subjectIdsByName: ReadonlyMap): Map [id, name])) } +/** Placeholder study for export metadata when an export spans Studies or has none (ADR 0011). */ +const MULTI_STUDY_RECORD: StudyRecord = { + id: 0, + shortCode: 'MULTI', + title: 'Multi-Study Export', + description: null, + leadPerson: '', + isLongitudinal: false, + created: '', + lastUpdated: '', + createdBy: null, + updatedBy: null, +} + +/** The filters and Study a JSON export records in its metadata. */ +interface JsonExportMetadata { + filters: ExportFilters + study: StudyRecord +} + +/** JSON export metadata for an export that spans Studies or has none. */ +const MULTI_STUDY_METADATA: JsonExportMetadata = { filters: { study: 'MULTI' }, study: MULTI_STUDY_RECORD } + +type FormattedContainerExport = + | { kind: 'json'; json: Awaited> } + | { kind: 'csv'; csv: string } + | { kind: 'xlsx'; xlsx: Buffer } + async function formatContainerExport( database: Database, data: ContainerExportData[], - format: ContainerExportFormat, - filters: ExportFilters, - study: StudyRecord, - columns: string[] | undefined, - csvOptions: CSVExportOptions | undefined, - userId: number | null | undefined -): Promise<{ csv?: string; xlsx?: Buffer; jsonData?: ContainerExportData[] }> { + params: { + format: ContainerExportFormat + columns: string[] | undefined + csvOptions: CSVExportOptions | undefined + userId: number | null | undefined + /** Defaults to the multi-study placeholder. */ + metadata?: JsonExportMetadata + } +): Promise { + const { format, columns, csvOptions, userId, metadata = MULTI_STUDY_METADATA } = params if (format === 'json') { - const json = await formatAsJSON(database, data, filters, study, columns, userId) - return { jsonData: json.containers } + const json = await formatAsJSON(database, data, metadata.filters, metadata.study, columns, userId) + return { kind: 'json', json } } if (format === 'csv') { - const csv = await formatAsCSV(database, data, columns, csvOptions, userId) - return { csv } + return { kind: 'csv', csv: await formatAsCSV(database, data, columns, csvOptions, userId) } } - const xlsx = await formatAsExcel(database, data, columns, userId) - return { xlsx } + return { kind: 'xlsx', xlsx: await formatAsExcel(database, data, columns, userId) } +} + +/** A formatted export's data, as `buildPostExportEnvelope` takes it. */ +function postEnvelopeData( + formatted: FormattedContainerExport, +): { jsonData?: ContainerExportData[]; csv?: string; xlsx?: Buffer } { + switch (formatted.kind) { + case 'json': + return { jsonData: formatted.json.containers } + case 'csv': + return { csv: formatted.csv } + case 'xlsx': + return { xlsx: formatted.xlsx } + } } export async function runSingleStudyContainerExportGet( @@ -157,12 +200,7 @@ export async function runSingleStudyContainerExportGet( userId?: number | null countOnly?: boolean } -): Promise< - | { kind: 'count'; count: number } - | { kind: 'csv'; csv: string; filename: string } - | { kind: 'xlsx'; xlsx: Buffer; filename: string } - | { kind: 'json'; json: Awaited>; filename: string } -> { +): Promise<{ kind: 'count'; count: number } | (FormattedContainerExport & { filename: string })> { const { filters, format, columns, csvOptions, userId, countOnly } = params const studyRecord = await resolveStudyRecordByShortCode(database, filters.study) if (!studyRecord) { @@ -193,22 +231,15 @@ export async function runSingleStudyContainerExportGet( } const enrichedData = await enrichContainerData(database, result.containers) - - const timestamp = formatExportTimestamp() - const filenameBase = `study_${studyRecord.shortCode}_export_${timestamp}` - - if (format === 'json') { - const json = await formatAsJSON(database, enrichedData, filters, studyRecord, columns, userId) - return { kind: 'json', json, filename: `${filenameBase}.json` } - } - - if (format === 'csv') { - const csv = await formatAsCSV(database, enrichedData, columns, csvOptions, userId) - return { kind: 'csv', csv, filename: `${filenameBase}.csv` } - } - - const xlsx = await formatAsExcel(database, enrichedData, columns, userId) - return { kind: 'xlsx', xlsx, filename: `${filenameBase}.xlsx` } + const formatted = await formatContainerExport(database, enrichedData, { + format, + columns, + csvOptions, + userId, + metadata: { filters, study: studyRecord }, + }) + const filenameBase = `study_${studyRecord.shortCode}_export_${formatExportTimestamp()}` + return { ...formatted, filename: `${filenameBase}.${formatted.kind}` } } export async function runSingleStudyContainerExportPost( @@ -304,39 +335,22 @@ export async function runSingleStudyContainerExportPost( subjectIdToName ) - const formatted = await formatContainerExport( - database, - enrichedData, + const formatted = await formatContainerExport(database, enrichedData, { format, - { ...filters, study: studyCode, date_tolerance: dateTolerance }, - studyRecord, columns, csvOptions, - userId - ) + userId, + metadata: { filters: { ...filters, study: studyCode, date_tolerance: dateTolerance }, study: studyRecord }, + }) return buildPostExportEnvelope({ format, filenameBase: `study_${studyRecord.shortCode}_export_${formatExportTimestamp()}`, summary, - ...formatted, + ...postEnvelopeData(formatted), }) } -/** Placeholder study for export metadata when an export spans Studies or has none (ADR 0011). */ -const MULTI_STUDY_RECORD: StudyRecord = { - id: 0, - shortCode: 'MULTI', - title: 'Multi-Study Export', - description: null, - leadPerson: '', - isLongitudinal: false, - created: '', - lastUpdated: '', - createdBy: null, - updatedBy: null, -} - export async function runMultiStudyContainerExportPost( database: Database, params: { @@ -365,17 +379,8 @@ export async function runMultiStudyContainerExportPost( errors: [`Invalid study codes: ${validation.invalid.join(', ')}`], } if (countOnly) return { count: 0, summary } - const formatted = await formatContainerExport( - database, - [], - format, - { study: 'MULTI' }, - MULTI_STUDY_RECORD, - columns, - csvOptions, - userId - ) - return buildPostExportEnvelope({ format, filenameBase, summary, ...formatted }) + const formatted = await formatContainerExport(database, [], { format, columns, csvOptions, userId }) + return buildPostExportEnvelope({ format, filenameBase, summary, ...postEnvelopeData(formatted) }) } const entriesByStudy = new Map() @@ -447,18 +452,9 @@ export async function runMultiStudyContainerExportPost( return { count: enrichedData.length, summary } } - const formatted = await formatContainerExport( - database, - enrichedData, - format, - { study: 'MULTI' }, - MULTI_STUDY_RECORD, - columns, - csvOptions, - userId - ) + const formatted = await formatContainerExport(database, enrichedData, { format, columns, csvOptions, userId }) - return buildPostExportEnvelope({ format, filenameBase, summary, ...formatted }) + return buildPostExportEnvelope({ format, filenameBase, summary, ...postEnvelopeData(formatted) }) } export async function runBarcodeContainerExportPost( @@ -479,16 +475,7 @@ export async function runBarcodeContainerExportPost( const selected = await selectContainers(database, { kind: 'micronixBarcodes', barcodes: foundBarcodes }) const enrichedData = await enrichContainerData(database, selected.containers) - const formatted = await formatContainerExport( - database, - enrichedData, - format, - { study: 'MULTI' }, - MULTI_STUDY_RECORD, - columns, - csvOptions, - userId - ) + const formatted = await formatContainerExport(database, enrichedData, { format, columns, csvOptions, userId }) return buildPostExportEnvelope({ format, @@ -498,7 +485,7 @@ export async function runBarcodeContainerExportPost( barcodes_found: foundBarcodes, barcodes_not_found: barcodes.filter((barcode) => !barcodeToContainerId.has(barcode)), }, - ...formatted, + ...postEnvelopeData(formatted), }) } From 07dbca6faa0cf66f445cb8d3a75fbd171fbc0bf8 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:25:52 -0700 Subject: [PATCH 25/28] refactor(api): derive ContainerSubtype from one list of Container types. Container selection's CONTAINER_TYPES re-spelled the ContainerSubtype union. container-placement now exports CONTAINER_SUBTYPES and derives the type from it, and the module filters on that list. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/container-placement.ts | 5 ++++- packages/api/src/lib/container-selection/filters.ts | 13 ++----------- packages/api/src/lib/container-selection/index.ts | 7 +++---- packages/api/src/lib/container-selection/types.ts | 2 +- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/packages/api/src/lib/container-placement.ts b/packages/api/src/lib/container-placement.ts index 81209f2..9fb1efe 100644 --- a/packages/api/src/lib/container-placement.ts +++ b/packages/api/src/lib/container-placement.ts @@ -13,7 +13,10 @@ import { } from '../db/schema' import { eq, inArray } from 'drizzle-orm' -export type ContainerSubtype = 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' | 'unknown' +/** Every Container type, in display order. `'unknown'` is a Container with no subtype row. */ +export const CONTAINER_SUBTYPES = ['micronix_tube', 'cryovial_tube', 'paper', 'static_well', 'unknown'] as const + +export type ContainerSubtype = (typeof CONTAINER_SUBTYPES)[number] export type CollectionType = 'micronix_plate' | 'cryovial_box' | 'sheet' diff --git a/packages/api/src/lib/container-selection/filters.ts b/packages/api/src/lib/container-selection/filters.ts index 4290279..64012b0 100644 --- a/packages/api/src/lib/container-selection/filters.ts +++ b/packages/api/src/lib/container-selection/filters.ts @@ -1,22 +1,13 @@ -import type { ContainerSubtype } from '../container-placement' +import { CONTAINER_SUBTYPES, type ContainerSubtype } from '../container-placement' import type { SpecimenSourceType } from './types' -/** Every Container type, in display order. `'unknown'` is a Container with no subtype row. */ -export const CONTAINER_TYPES = [ - 'micronix_tube', - 'cryovial_tube', - 'paper', - 'static_well', - 'unknown', -] as const satisfies readonly ContainerSubtype[] - /** * Narrow wire container types to the Container types filter. No values means every type; * unrecognised values are dropped, so a list of only unrecognised values matches nothing. */ export function containerTypesFilter(values: readonly string[] | undefined): ContainerSubtype[] | undefined { if (!values || values.length === 0) return undefined - return CONTAINER_TYPES.filter((type) => values.includes(type)) + return CONTAINER_SUBTYPES.filter((type) => values.includes(type)) } /** Narrow a wire source type; anything other than `'subject'` or `'control'` means both. */ diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 06c46fc..9ffea48 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -6,7 +6,7 @@ import { and, eq, getTableColumns, inArray, type SQL } from 'drizzle-orm' import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject } from '../../db/schema' -import type { ContainerSubtype } from '../container-placement' +import { CONTAINER_SUBTYPES, type ContainerSubtype } from '../container-placement' import { containerFilterConditions, containerTypeSql, @@ -15,7 +15,6 @@ import { specimenFilterConditions, } from './conditions' import { micronixBarcodeScopes, subjectListScopes, type SelectionScope } from './scopes' -import { CONTAINER_TYPES } from './filters' import type { ContainerSelection, ContainerSelectionResult, @@ -26,7 +25,7 @@ import type { export * from './types' export { containerHasAllTags, hasContainerLevelFilter } from './conditions' -export { CONTAINER_TYPES, containerTypesFilter, specimenSourceType } from './filters' +export { containerTypesFilter, specimenSourceType } from './filters' /** Select the Containers and Specimens a selection matches, with each Container's type. */ export async function selectContainers( @@ -69,7 +68,7 @@ export async function selectContainerSelectionTypes( } return { specimenTypeIds: [...specimenTypeIds].sort((a, b) => a - b), - containerTypes: CONTAINER_TYPES.filter((type) => containerTypes.has(type)), + containerTypes: CONTAINER_SUBTYPES.filter((type) => containerTypes.has(type)), } } diff --git a/packages/api/src/lib/container-selection/types.ts b/packages/api/src/lib/container-selection/types.ts index d63c48b..fef36b1 100644 --- a/packages/api/src/lib/container-selection/types.ts +++ b/packages/api/src/lib/container-selection/types.ts @@ -82,6 +82,6 @@ export interface ContainerSelectionResult { export interface ContainerSelectionTypes { /** Specimen type ids, ascending. */ specimenTypeIds: number[] - /** Container types, in `CONTAINER_TYPES` order; `'unknown'` when a Container has no subtype row. */ + /** Container types, in `CONTAINER_SUBTYPES` order; `'unknown'` when a Container has no subtype row. */ containerTypes: ContainerSubtype[] } From 6f451f6efd1f02eaf589c530f297058043e4e037 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:26:43 -0700 Subject: [PATCH 26/28] refactor(api): report a Container-level filter on the selection result. hasContainerLevelFilter built every Container-level condition just to count them, and dashboard statistics rebuilt the selection filters to call it. The selection result now says whether one was set. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/container-selection.test.ts | 10 ++++++---- packages/api/src/lib/container-selection/conditions.ts | 8 -------- packages/api/src/lib/container-selection/index.ts | 9 +++++++-- packages/api/src/lib/container-selection/types.ts | 5 +++++ packages/api/src/lib/statistics/dashboard-stats.ts | 5 ++--- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts index 6a101d0..95b9121 100644 --- a/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts +++ b/packages/api/src/lib/container-selection/__tests__/container-selection.test.ts @@ -133,6 +133,7 @@ describe('selectContainers', () => { expect(ids(result.specimens)).toEqual([stored.id, unstored.id]) expect(result.containers).toHaveLength(1) + expect(result.hasContainerLevelFilter).toBe(false) }) it('matches nothing for an unknown Study id', async () => { @@ -141,7 +142,7 @@ describe('selectContainers', () => { const result = await selectContainers(db, { kind: 'study', studyId: study.id + 999 }) - expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map(), hasContainerLevelFilter: false }) }) }) @@ -214,6 +215,7 @@ describe('selectContainers', () => { expect(ids(result.containers)).toEqual([januaryTube.id]) expect(ids(result.specimens)).toEqual([january.id]) + expect(result.hasContainerLevelFilter).toBe(true) }) it('keeps Containers of the listed container types, and only their Specimens', async () => { @@ -244,7 +246,7 @@ describe('selectContainers', () => { const result = await selectContainers(db, select(study.id, subject.id, { containerTypes: containerTypesFilter(['not_a_type']) })) - expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map(), hasContainerLevelFilter: true }) }) it('keeps Containers carrying every selected Tag, and only their Specimens', async () => { @@ -312,7 +314,7 @@ describe('selectContainers', () => { expect(ids(underRack.containers)).toEqual([tube.id, vial.id, well.id, boxedPaper.id, baggedPaper.id]) expect(ids(underRack.specimens)).toEqual([specimen.id]) expect(ids(underShelf.containers)).toEqual([tube.id, well.id, boxedPaper.id]) - expect(unknown).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + expect(unknown).toEqual({ containers: [], specimens: [], containerTypes: new Map(), hasContainerLevelFilter: true }) }) }) @@ -432,7 +434,7 @@ describe('selectContainers', () => { const result = await selectContainers(db, { kind: 'subjects', subjects: [] }) - expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map() }) + expect(result).toEqual({ containers: [], specimens: [], containerTypes: new Map(), hasContainerLevelFilter: false }) }) it('selects every listed Subject when the list is longer than one IN chunk', async () => { diff --git a/packages/api/src/lib/container-selection/conditions.ts b/packages/api/src/lib/container-selection/conditions.ts index be81a33..1e66d99 100644 --- a/packages/api/src/lib/container-selection/conditions.ts +++ b/packages/api/src/lib/container-selection/conditions.ts @@ -79,14 +79,6 @@ export function containerFilterConditions(filters: ContainerSelectionFilters): S return conditions } -/** - * Whether any Container-level filter is set: Container created, container types, Tags or - * Location. With one set, the selection keeps only Specimens with a matching Container. - */ -export function hasContainerLevelFilter(filters: ContainerSelectionFilters): boolean { - return containerFilterConditions(filters).length > 0 -} - /** * Containers stored in a Location or any Location under it: tubes and static wells on * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags there. diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 9ffea48..9632712 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -24,7 +24,7 @@ import type { } from './types' export * from './types' -export { containerHasAllTags, hasContainerLevelFilter } from './conditions' +export { containerHasAllTags } from './conditions' export { containerTypesFilter, specimenSourceType } from './filters' /** Select the Containers and Specimens a selection matches, with each Container's type. */ @@ -143,5 +143,10 @@ async function runScopes( specimens = specimens.filter((row) => stored.has(row.id)) } const byId = (a: { id: number }, b: { id: number }) => a.id - b.id - return { containers: containers.sort(byId), specimens: specimens.sort(byId), containerTypes } + return { + containers: containers.sort(byId), + specimens: specimens.sort(byId), + containerTypes, + hasContainerLevelFilter: containerConditions.length > 0, + } } diff --git a/packages/api/src/lib/container-selection/types.ts b/packages/api/src/lib/container-selection/types.ts index fef36b1..656c7be 100644 --- a/packages/api/src/lib/container-selection/types.ts +++ b/packages/api/src/lib/container-selection/types.ts @@ -76,6 +76,11 @@ export interface ContainerSelectionResult { specimens: SelectedSpecimen[] /** Each selected Container's type, by Container id. */ containerTypes: Map + /** + * Whether a Container-level filter (Container created, container types, Tags or Location) + * was set, so `specimens` holds only Specimens with a matching Container. + */ + hasContainerLevelFilter: boolean } /** The specimen types and container types a selection matches. */ diff --git a/packages/api/src/lib/statistics/dashboard-stats.ts b/packages/api/src/lib/statistics/dashboard-stats.ts index 2fa83f5..7a62a27 100644 --- a/packages/api/src/lib/statistics/dashboard-stats.ts +++ b/packages/api/src/lib/statistics/dashboard-stats.ts @@ -1,7 +1,6 @@ import type { Database } from '../../db/client' import { containerTypesFilter, - hasContainerLevelFilter, selectContainers, specimenSourceType, type ContainerSelection, @@ -82,11 +81,11 @@ export async function getDashboardStatistics( ): Promise { const selection = await statisticsSelection(database, filters) if (!selection) return emptyDashboardStatistics() - const { containers, specimens, containerTypes } = await selectContainers(database, selection) + const { containers, specimens, containerTypes, hasContainerLevelFilter } = await selectContainers(database, selection) if (specimens.length === 0) return emptyDashboardStatistics() const specimenAggregates = await computeSpecimenAggregates(database, specimens, { - useSpecimenTypeCache: !hasContainerLevelFilter(selectionFilters(filters)), + useSpecimenTypeCache: !hasContainerLevelFilter, }) const containerTotal = containers.length const averagePerSpecimen = specimenAggregates.total > 0 ? containerTotal / specimenAggregates.total : 0 From d3dfd8a210662fcb2b65c03a030d722b5dbe3277 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:27:21 -0700 Subject: [PATCH 27/28] refactor(statistics): parse filter ids with the rest of the statistics query. dashboard-stats parsed specimen_type_id and location_id from wire strings although parseStatisticsFilters parses the query. StatisticsFilters now carries them as numbers. Co-Authored-By: Claude Opus 5.5 --- .../statistics/__tests__/dashboard-stats.test.ts | 2 +- .../api/src/lib/statistics/dashboard-stats.ts | 12 ++---------- packages/api/src/lib/statistics/parse-filters.ts | 11 +++++++++-- packages/api/src/lib/statistics/types.ts | 4 ++-- .../api/src/routes/__tests__/statistics.test.ts | 15 +++++++++++++++ 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts b/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts index 9cadc97..69a47c1 100644 --- a/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts +++ b/packages/api/src/lib/statistics/__tests__/dashboard-stats.test.ts @@ -150,7 +150,7 @@ describe('dashboard-stats', () => { }) const data = await getDashboardStatistics(testDb, { - location_id: String(loc.id), + location_id: loc.id, }) expect(data.containers.total).toBe(1) diff --git a/packages/api/src/lib/statistics/dashboard-stats.ts b/packages/api/src/lib/statistics/dashboard-stats.ts index 7a62a27..b76ca2d 100644 --- a/packages/api/src/lib/statistics/dashboard-stats.ts +++ b/packages/api/src/lib/statistics/dashboard-stats.ts @@ -37,23 +37,15 @@ function emptyDashboardStatistics(): DashboardStatistics { } } -/** Parse an integer id query value; anything else means no filter. */ -function parseId(value: string | undefined): number | undefined { - if (!value) return undefined - const id = parseInt(value) - return isNaN(id) ? undefined : id -} - /** Map the statistics query to Container selection filters (ADR 0011). */ function selectionFilters(filters: StatisticsFilters): ContainerSelectionFilters { - const specimenTypeId = parseId(filters.specimen_type_id) return { - specimenTypeIds: specimenTypeId === undefined ? undefined : [specimenTypeId], + specimenTypeIds: filters.specimen_type_id === undefined ? undefined : [filters.specimen_type_id], collectionDate: { from: filters.collection_date_from, to: filters.collection_date_to }, specimenCreated: { from: filters.created_from, to: filters.created_to }, containerTypes: containerTypesFilter(filters.container_type ? [filters.container_type] : undefined), tagIds: filters.tag_ids, - locationId: parseId(filters.location_id), + locationId: filters.location_id, } } diff --git a/packages/api/src/lib/statistics/parse-filters.ts b/packages/api/src/lib/statistics/parse-filters.ts index 96259c1..f8dc3b0 100644 --- a/packages/api/src/lib/statistics/parse-filters.ts +++ b/packages/api/src/lib/statistics/parse-filters.ts @@ -1,6 +1,13 @@ import type { Context } from 'hono' import type { StatisticsFilters } from './types' +/** Parse an integer id query value; anything else means no filter. */ +function parseId(value: string | undefined): number | undefined { + if (!value) return undefined + const id = parseInt(value) + return isNaN(id) ? undefined : id +} + /** Parse dashboard statistics query parameters from a Hono request. */ export function parseStatisticsFilters(c: Context): StatisticsFilters { const tagIdsParam = c.req.queries('tag_ids') || c.req.queries('tag_ids[]') @@ -9,10 +16,10 @@ export function parseStatisticsFilters(c: Context): StatisticsFilters { return { study: c.req.query('study'), source_type: c.req.query('source_type'), - specimen_type_id: c.req.query('specimen_type_id'), + specimen_type_id: parseId(c.req.query('specimen_type_id')), container_type: c.req.query('container_type'), tag_ids: tagIds, - location_id: c.req.query('location_id'), + location_id: parseId(c.req.query('location_id')), collection_date_from: c.req.query('collection_date_from'), collection_date_to: c.req.query('collection_date_to'), created_from: c.req.query('created_from'), diff --git a/packages/api/src/lib/statistics/types.ts b/packages/api/src/lib/statistics/types.ts index e5659cc..dacc592 100644 --- a/packages/api/src/lib/statistics/types.ts +++ b/packages/api/src/lib/statistics/types.ts @@ -1,10 +1,10 @@ export type StatisticsFilters = { study?: string source_type?: string - specimen_type_id?: string + specimen_type_id?: number container_type?: string tag_ids?: number[] - location_id?: string + location_id?: number collection_date_from?: string collection_date_to?: string created_from?: string diff --git a/packages/api/src/routes/__tests__/statistics.test.ts b/packages/api/src/routes/__tests__/statistics.test.ts index e927c04..7b69e89 100644 --- a/packages/api/src/routes/__tests__/statistics.test.ts +++ b/packages/api/src/routes/__tests__/statistics.test.ts @@ -5,6 +5,7 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createStatisticsRoutes } from '../statistics' +import { createTestSpecimenType, createTestStorageContainer } from '../../__tests__/helpers/factories' describe('Statistics API', () => { let ctx: AuthenticatedRouteTestContext @@ -47,6 +48,20 @@ describe('Statistics API', () => { expect(data).toBeDefined() }) + it('filters by a numeric specimen_type_id and ignores a non-numeric one or location_id', async () => { + await createTestStorageContainer(ctx.db) + const otherType = await createTestSpecimenType(ctx.db, { name: 'Other type' }) + const total = async (query: string) => { + const res = await ctx.request(`/api/statistics${query}`, { method: 'GET' }) + expect(res.status).toBe(200) + return ((await res.json()) as { specimens: { total: number } }).specimens.total + } + + expect(await total('')).toBe(1) + expect(await total(`?specimen_type_id=${otherType.id}`)).toBe(0) + expect(await total('?specimen_type_id=abc&location_id=abc')).toBe(1) + }) + it('returns 401 without auth', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/statistics', { method: 'GET' }) expect(res.status).toBe(401) From 6e14b17d9ee49ce86ff2469062d9100b4ad8b8b8 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Thu, 29 Oct 2026 22:44:38 -0700 Subject: [PATCH 28/28] refactor(api): derive the remaining container type unions from CONTAINER_SUBTYPES. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/container-api-enrichment.ts | 3 ++- packages/api/src/lib/container-creation.ts | 4 +++- packages/api/src/lib/container-move.ts | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/api/src/lib/container-api-enrichment.ts b/packages/api/src/lib/container-api-enrichment.ts index 1c6d838..1008ffd 100644 --- a/packages/api/src/lib/container-api-enrichment.ts +++ b/packages/api/src/lib/container-api-enrichment.ts @@ -5,12 +5,13 @@ import type { MicronixSubtypeDetails, PaperSubtypeDetails, StaticWellSubtypeDetails, + ContainerSubtype, } from './container-placement' import { loadContainerReadViews, type ContainerReadView } from './container-read-view' import type { CollectionInfo } from '../types/collections' export type EnrichedContainerApi = StorageContainer & { - containerType: 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' | 'unknown' + containerType: ContainerSubtype tags: Array<{ id: number; name: string }> unit: Unit | undefined location: Location | null diff --git a/packages/api/src/lib/container-creation.ts b/packages/api/src/lib/container-creation.ts index 8ffd31f..7a3c44c 100644 --- a/packages/api/src/lib/container-creation.ts +++ b/packages/api/src/lib/container-creation.ts @@ -3,8 +3,10 @@ import type { ContainerWriteInput } from '@sampledb/contract' import type { DatabaseOrTransaction } from './db-types' import { registerSpecimens, SpecimenRegistrationError } from './specimen-registration' import { ConflictError } from './error-handler' +import type { ContainerSubtype } from './container-placement' -export type ContainerType = 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' +/** A Container subtype a write can name; `unknown` is only read back. */ +export type ContainerType = Exclude export type ContainerQuantity = { unitId?: number diff --git a/packages/api/src/lib/container-move.ts b/packages/api/src/lib/container-move.ts index 55c6ddd..15138b9 100644 --- a/packages/api/src/lib/container-move.ts +++ b/packages/api/src/lib/container-move.ts @@ -14,6 +14,7 @@ import { eq, and, isNull } from 'drizzle-orm' import { resolveCollection, resolveSheetsByName, type CollectionType } from './collections/collection-resolve' import { checkGridPositionOccupancy } from './container-occupancy' import { normalizePosition } from './normalize-position' +import type { ContainerSubtype } from './container-placement' /** Staging prefix for position updates so swaps on one plate do not violate UNIQUE(collection_id, position) mid-transaction. */ const MOVE_STAGING_PREFIX = '__mv_' @@ -22,7 +23,8 @@ function stagingPositionForContainer(containerId: number): string { return `${MOVE_STAGING_PREFIX}${containerId}` } -export type ContainerType = 'micronix_tube' | 'cryovial_tube' | 'static_well' | 'paper' +/** A Container subtype a write can name; `unknown` is only read back. */ +export type ContainerType = Exclude export interface ContainerInfo { containerId: number