From 5448ddbeef3346d6067e2db62c591f981b4d447e Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:29:01 -0700 Subject: [PATCH 01/54] fix(containers): save a Container edit atomically and edit Paper sublabels (#108) PATCH /containers/:id now checks the Container, unit, barcode or sublabel, and Tags before any write, and runs the checks and writes in one withWriteTransaction. Tag ids are de-duplicated; an unknown Tag is a 400. A failed edit, including a UNIQUE failure on the last write, leaves the Container unchanged. Errors go through handleRouteError, so they are logged. The edit modal shows a Sublabel field for Paper and sends it as `sublabel` instead of `barcode`, which the API rejects. It saves through containersApi.update and a new useUpdateContainer hook (invalidates every container query) instead of raw fetch. Co-Authored-By: Claude Opus 5.5 --- .../src/routes/__tests__/containers.test.ts | 195 +++++++++++- packages/api/src/routes/containers.ts | 281 +++++++++--------- .../web/src/components/ContainerEditModal.tsx | 74 +++-- .../__tests__/ContainerEditModal.test.tsx | 117 +++++++- .../hooks/__tests__/useContainers.test.tsx | 52 ++++ packages/web/src/hooks/useContainers.ts | 17 +- .../src/lib/api/__tests__/containers.test.ts | 25 ++ packages/web/src/lib/api/containers.ts | 19 ++ packages/web/src/lib/api/parse-response.ts | 8 + packages/web/src/pages/ContainerDetail.tsx | 8 +- 10 files changed, 602 insertions(+), 194 deletions(-) create mode 100644 packages/web/src/hooks/__tests__/useContainers.test.tsx diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index c56dd06a..cd9aa39d 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -7,6 +7,7 @@ import { import { createContainersRoutes } from '../containers' import * as containerEnrichment from '../../lib/container-api-enrichment' import * as containerReadView from '../../lib/container-read-view' +import * as identifierResolution from '../../lib/identifier-resolution' import { utcNow } from '../../lib/datetime' import { createTestStudy, @@ -20,8 +21,14 @@ import { createTestCryovialTube, createTestStorageContainer, createTestTag, + createTestContainerTags, + createTestMicronixPlate, + createTestMicronixTube, + createTestBox, + createTestSheet, + createTestPaper, } from '../../__tests__/helpers/factories' -import { cryovialBox, cryovialTube, specimen, storageContainer, micronixPlate, micronixTube, box, sheet, paper, staticWell, storageContainerTag } from '../../db/schema' +import { cryovialBox, cryovialTube, specimen, storageContainer, micronixPlate, micronixTube, box, sheet, paper, staticWell, storageContainerTag, errorLogs } from '../../db/schema' import { eq } from 'drizzle-orm' describe('Containers API', () => { @@ -396,4 +403,190 @@ describe('Containers API', () => { expect(row?.sublabel).toBe('P-NEW') }) }) + describe('PATCH /api/containers/:id saves every change or none', () => { + let unitId: number + let specimenId: number + let locationId: number + let keepTagId: number + let otherTagId: number + + beforeEach(async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer', description: 'Test' }) + const location = await createTestLocation(ctx.db, { + name: 'Atomic Loc', + parentId: null, + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + locationId = location.id + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + unitId = unit.id + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Whole Blood' }) + specimenId = (await createTestSpecimen(ctx.db, specimenType.id)).id + keepTagId = (await createTestTag(ctx.db, { name: 'keep' })).id + otherTagId = (await createTestTag(ctx.db, { name: 'other' })).id + }) + + /** A Container with comment 'original', quantity 1 and the 'keep' Tag. */ + async function seedContainer() { + const container = await createTestStorageContainer(ctx.db, { specimenId, unitId }) + await ctx.db.update(storageContainer).set({ comment: 'original' }).where(eq(storageContainer.id, container.id)) + await createTestContainerTags(ctx.db, container.id, [keepTagId]) + return container.id + } + + async function seedMicronix(barcode: string, position: string) { + const plate = await createTestMicronixPlate(ctx.db, { name: `PL-${barcode}`, locationId }) + const id = await seedContainer() + await createTestMicronixTube(ctx.db, { containerId: id, collectionId: plate.id, barcode, position }) + return id + } + + async function seedPaper(sublabel: string) { + const boxRecord = await createTestBox(ctx.db, { name: `BX-${sublabel}`, locationId }) + const sheetRecord = await createTestSheet(ctx.db, { name: 'S1', boxId: boxRecord.id }) + const id = await seedContainer() + await createTestPaper(ctx.db, { containerId: id, sheetId: sheetRecord.id, sublabel }) + return id + } + + async function tagIdsOf(id: number) { + const rows = await ctx.db + .select({ tagId: storageContainerTag.tagId }) + .from(storageContainerTag) + .where(eq(storageContainerTag.storageContainerId, id)) + return rows.map((row) => row.tagId).sort() + } + + async function expectUnchanged(id: number) { + const row = await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, id)).get() + expect(row?.comment).toBe('original') + expect(row?.remainingQuantity).toBe(1) + expect(await tagIdsOf(id)).toEqual([keepTagId]) + } + + it('saves one Tag when the same Tag is sent twice', async () => { + const id = await seedMicronix('MX-DUP', 'A01') + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { tagIds: [otherTagId, otherTagId] } }) + + expect(res.status).toBe(200) + expect(await tagIdsOf(id)).toEqual([otherTagId]) + const body = (await res.json()) as { container: { tags?: Array<{ id: number }> } } + expect(body.container.tags?.map((t) => t.id)).toEqual([otherTagId]) + }) + + it('returns 400 for an unknown Tag and leaves the Container unchanged', async () => { + const id = await seedMicronix('MX-UNK', 'A01') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', remainingQuantity: 0.5, tagIds: [otherTagId, 9999] }, + }) + + expect(res.status).toBe(400) + const body = (await res.json()) as { error: string; errorCode: string } + expect(body.error).toBe('Unknown tag id: 9999') + expect(body.errorCode).toBe('VALIDATION_ERROR') + await expectUnchanged(id) + }) + + it('returns 400 for a barcode already in use and leaves quantity, comment and Tags unchanged', async () => { + await seedMicronix('MX-TAKEN', 'A01') + const id = await seedMicronix('MX-MINE', 'A01') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', remainingQuantity: 0.5, tagIds: [otherTagId], barcode: 'MX-TAKEN' }, + }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe("Barcode 'MX-TAKEN' is already in use") + await expectUnchanged(id) + const tube = await ctx.db.select().from(micronixTube).where(eq(micronixTube.id, id)).get() + expect(tube?.barcode).toBe('MX-MINE') + }) + + it('rolls back every change when the barcode write itself fails', async () => { + await seedMicronix('MX-RACE', 'A01') + const id = await seedMicronix('MX-SELF', 'A01') + // The check misses the conflict (as when another request takes the barcode first), so the + // UNIQUE constraint fails on the last write of the edit. + vi.spyOn(identifierResolution, 'resolveContainerByBarcode').mockResolvedValue(null) + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', remainingQuantity: 0.5, tagIds: [otherTagId], barcode: 'MX-RACE' }, + }) + + expect(res.status).toBe(409) + expect(((await res.json()) as { errorCode: string }).errorCode).toBe('CONFLICT') + await expectUnchanged(id) + }) + + it('returns 400 for a Paper sublabel already in use and leaves the Container unchanged', async () => { + await seedPaper('SPOT-TAKEN') + const id = await seedPaper('SPOT-MINE') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', remainingQuantity: 0.5, tagIds: [otherTagId], sublabel: 'SPOT-TAKEN' }, + }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe("Sublabel 'SPOT-TAKEN' is already in use") + await expectUnchanged(id) + const row = await ctx.db.select().from(paper).where(eq(paper.id, id)).get() + expect(row?.sublabel).toBe('SPOT-MINE') + }) + + it('returns 400 when a Paper Container is sent a barcode and leaves it unchanged', async () => { + const id = await seedPaper('SPOT-BC') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', tagIds: [otherTagId], barcode: 'SPOT-NEW' }, + }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe( + 'Paper containers use sublabel for spot identifiers, not barcode', + ) + await expectUnchanged(id) + }) + + it('returns 400 for an invalid body through the standard handler', async () => { + const id = await seedMicronix('MX-ZOD', 'A01') + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { tagIds: ['one'] } }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { errorCode: string }).errorCode).toBe('VALIDATION_ERROR') + await expectUnchanged(id) + }) + + it('returns 404 for a Container that does not exist', async () => { + const res = await ctx.request('/api/containers/999999', { method: 'PATCH', json: { comment: 'x' } }) + + expect(res.status).toBe(404) + expect(((await res.json()) as { errorCode: string }).errorCode).toBe('NOT_FOUND') + }) + + it('logs a server error to error_logs through the standard handler', async () => { + const id = await seedMicronix('MX-500', 'A01') + vi.spyOn(containerEnrichment, 'enrichContainerForApi').mockRejectedValueOnce(new Error('enrichment failed')) + const logsBefore = await ctx.db.select().from(errorLogs) + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { comment: 'changed' } }) + + expect(res.status).toBe(500) + const body = (await res.json()) as { error: string; errorCode: string } + expect(body.error).toBe('enrichment failed') + // The handler writes the log without awaiting it. + await new Promise((resolve) => setTimeout(resolve, 50)) + const logsAfter = await ctx.db.select().from(errorLogs) + expect(logsAfter.length).toBe(logsBefore.length + 1) + expect(logsAfter.at(-1)?.message).toBe('enrichment failed') + }) + }) }) diff --git a/packages/api/src/routes/containers.ts b/packages/api/src/routes/containers.ts index 71539db5..eaeeb44f 100644 --- a/packages/api/src/routes/containers.ts +++ b/packages/api/src/routes/containers.ts @@ -1,7 +1,7 @@ import { Hono } from 'hono' import type { Database } from '../db/client' import { storageContainer, micronixTube, cryovialTube, paper, staticWell, tag, storageContainerTag } from '../db/schema' -import { eq, and, sql } from 'drizzle-orm' +import { eq, and, sql, inArray } from 'drizzle-orm' import { z } from 'zod' import { validatePage, validateLimit } from '../lib/constants' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' @@ -11,7 +11,15 @@ import { resolveContainerByBarcode } from '../lib/identifier-resolution' import { enrichContainerForApi, enrichContainersForApi, toEnrichedContainerApi } from '../lib/container-api-enrichment' import { loadContainerReadViews } from '../lib/container-read-view' import { mapEnrichedContainerToWire, mapEnrichedContainersToWire } from '../lib/container-wire-mapper' -import { handleRouteError, RouteError, containersFetchFailedBody } from '../lib/error-handler' +import { + handleRouteError, + RouteError, + containersFetchFailedBody, + NotFoundError, + ValidationError, +} from '../lib/error-handler' +import { withWriteTransaction } from '../db/write-transaction' +import { validateUnitForContainerType } from '../lib/validation' import { containerHasAllTags } from '../lib/container-selection' /** @@ -119,171 +127,152 @@ containers.get('/:id', authMiddleware, async (c) => { } }) -// Update container +// Update container. Every change is saved, or none: the checks and writes share one transaction. +const containerPatchSchema = z.object({ + comment: z.string().optional(), + remainingQuantity: z.number().optional(), + unitId: z.number().int().optional(), + tagIds: z.array(z.number().int()).optional(), // Replaces the Container's Tags + barcode: z.union([z.string(), z.null()]).optional(), + sublabel: z.union([z.string(), z.null()]).optional(), +}) + +type ContainerSubtype = 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' + +async function containerSubtypeOf(database: Database, id: number): Promise { + const [micronixInfo, cryovialInfo, paperInfo, staticWellInfo] = await Promise.all([ + database.select({ id: micronixTube.id }).from(micronixTube).where(eq(micronixTube.id, id)).get(), + database.select({ id: cryovialTube.id }).from(cryovialTube).where(eq(cryovialTube.id, id)).get(), + database.select({ id: paper.id }).from(paper).where(eq(paper.id, id)).get(), + database.select({ id: staticWell.id }).from(staticWell).where(eq(staticWell.id, id)).get(), + ]) + if (micronixInfo) return 'micronix_tube' + if (cryovialInfo) return 'cryovial_tube' + if (paperInfo) return 'paper' + if (staticWellInfo) return 'static_well' + return null +} + +/** The barcode to store (null clears it). Throws when this subtype can't take one. */ +function nextBarcode(containerType: ContainerSubtype | null, rawBarcode: string | null): string | null { + if (containerType === 'paper') { + throw new ValidationError('Paper containers use sublabel for spot identifiers, not barcode') + } + if (containerType === 'static_well' || !containerType) { + throw new ValidationError('Barcode cannot be set for this container type') + } + const trimmed = rawBarcode?.trim() ?? '' + if (containerType === 'micronix_tube') { + if (rawBarcode === null) throw new ValidationError('Micronix tube barcode cannot be cleared') + if (trimmed.length === 0) throw new ValidationError('Micronix tube barcode cannot be empty') + } + return trimmed.length === 0 ? null : trimmed +} + +function nextSublabel(containerType: ContainerSubtype | null, rawSublabel: string | null): string | null { + if (containerType !== 'paper') { + throw new ValidationError('Sublabel can only be set for paper containers') + } + const trimmed = rawSublabel?.trim() ?? '' + return trimmed.length === 0 ? null : trimmed +} + +async function assertIdentifierFree( + database: Database, + id: number, + identifier: string | null, + label: 'Barcode' | 'Sublabel', +): Promise { + if (identifier === null) return + const takenBy = await resolveContainerByBarcode(database, identifier) + if (takenBy !== null && takenBy !== id) { + throw new ValidationError(`${label} '${identifier}' is already in use`) + } +} + +/** De-duplicated Tag ids, after checking that each one exists. */ +async function knownTagIds(database: Database, tagIds: number[]): Promise { + const unique = [...new Set(tagIds)] + if (unique.length === 0) return unique + const found = await database.select({ id: tag.id }).from(tag).where(inArray(tag.id, unique)) + const foundIds = new Set(found.map((row) => row.id)) + const unknown = unique.filter((tagId) => !foundIds.has(tagId)) + if (unknown.length > 0) { + throw new ValidationError(`Unknown tag id${unknown.length > 1 ? 's' : ''}: ${unknown.join(', ')}`, { + unknownTagIds: unknown, + }) + } + return unique +} + containers.patch('/:id', memberMiddleware, async (c) => { try { const id = parseInt(requireParam(c, 'id')) - if (isNaN(id)) { - return c.json({ error: 'Invalid container ID' }, 400) - } - - const body = await c.req.json() - const schema = z.object({ - comment: z.string().optional(), - remainingQuantity: z.number().optional(), - unitId: z.number().int().optional(), - tagIds: z.array(z.number().int()).optional(), // Replace tags - barcode: z.union([z.string(), z.null()]).optional(), - sublabel: z.union([z.string(), z.null()]).optional(), - }) - - const data = schema.parse(body) - const { tagIds, unitId, barcode: rawBarcode, sublabel: rawSublabel, ...restData } = data - const updateData: { comment?: string | null; remainingQuantity?: number; unitId?: number } = { ...restData } - // An emptied comment clears it. - if (restData.comment !== undefined && restData.comment.trim() === '') updateData.comment = null - - const [container, micronixInfo, cryovialInfo, paperInfo, staticWellInfo] = await Promise.all([ - database.select().from(storageContainer).where(eq(storageContainer.id, id)).get(), - database.select().from(micronixTube).where(eq(micronixTube.id, id)).get(), - database.select().from(cryovialTube).where(eq(cryovialTube.id, id)).get(), - database.select().from(paper).where(eq(paper.id, id)).get(), - database.select().from(staticWell).where(eq(staticWell.id, id)).get(), - ]) - - if (!container) { - return c.json({ error: 'Container not found' }, 404) + throw new ValidationError('Invalid container ID') } - let containerType: 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' | null = null - if (micronixInfo) containerType = 'micronix_tube' - else if (cryovialInfo) containerType = 'cryovial_tube' - else if (paperInfo) containerType = 'paper' - else if (staticWellInfo) containerType = 'static_well' - - // Validate unit if provided - if (unitId !== undefined) { - if (containerType) { - // Validate unit is allowed for container type - const { validateUnitForContainerType } = await import('../lib/validation') - const validation = await validateUnitForContainerType(database, containerType, unitId) - if (!validation.valid) { - return c.json({ error: validation.error }, 400) - } - } - - updateData.unitId = unitId - } + const data = containerPatchSchema.parse(await c.req.json()) + const { tagIds, unitId, barcode: rawBarcode, sublabel: rawSublabel, comment, remainingQuantity } = data + const userId = c.get('user')?.id - let newBarcode: string | null | undefined - let newSublabel: string | null | undefined - if (rawBarcode !== undefined) { - if (containerType === 'paper') { - return c.json({ error: 'Paper containers use sublabel for spot identifiers, not barcode' }, 400) - } - if (containerType === 'static_well' || !containerType) { - return c.json({ error: 'Barcode cannot be set for this container type' }, 400) + const updated = await withWriteTransaction(database, async (tx) => { + const container = await tx.select({ id: storageContainer.id }).from(storageContainer).where(eq(storageContainer.id, id)).get() + if (!container) { + throw new NotFoundError('Container', id) } + const containerType = await containerSubtypeOf(tx, id) - if (containerType === 'micronix_tube') { - if (rawBarcode === null) { - return c.json({ error: 'Micronix tube barcode cannot be cleared' }, 400) - } - const trimmed = rawBarcode.trim() - if (trimmed.length === 0) { - return c.json({ error: 'Micronix tube barcode cannot be empty' }, 400) - } - newBarcode = trimmed - } else { - if (rawBarcode === null) { - newBarcode = null - } else { - const trimmed = rawBarcode.trim() - newBarcode = trimmed.length === 0 ? null : trimmed + // Check everything before the first write. + if (unitId !== undefined && containerType) { + const validation = await validateUnitForContainerType(tx, containerType, unitId) + if (!validation.valid) { + throw new ValidationError(validation.error ?? 'Invalid unit for this container type') } } - - if (newBarcode !== null) { - const takenBy = await resolveContainerByBarcode(database, newBarcode) - if (takenBy !== null && takenBy !== id) { - return c.json({ error: `Barcode '${newBarcode}' is already in use` }, 400) + const newBarcode = rawBarcode === undefined ? undefined : nextBarcode(containerType, rawBarcode) + if (newBarcode !== undefined) await assertIdentifierFree(tx, id, newBarcode, 'Barcode') + const newSublabel = rawSublabel === undefined ? undefined : nextSublabel(containerType, rawSublabel) + if (newSublabel !== undefined) await assertIdentifierFree(tx, id, newSublabel, 'Sublabel') + const newTagIds = tagIds === undefined ? undefined : await knownTagIds(tx, tagIds) + + const [row] = await tx + .update(storageContainer) + .set({ + ...(comment !== undefined && { comment: comment.trim() === '' ? null : comment }), + ...(remainingQuantity !== undefined && { remainingQuantity }), + ...(unitId !== undefined && { unitId }), + lastUpdated: utcNow(), + updatedBy: userId, + }) + .where(eq(storageContainer.id, id)) + .returning() + + if (newTagIds !== undefined) { + await tx.delete(storageContainerTag).where(eq(storageContainerTag.storageContainerId, id)) + if (newTagIds.length > 0) { + await tx.insert(storageContainerTag).values(newTagIds.map((tagId) => ({ storageContainerId: id, tagId }))) } } - } - if (rawSublabel !== undefined) { - if (containerType !== 'paper') { - return c.json({ error: 'Sublabel can only be set for paper containers' }, 400) - } - if (rawSublabel === null) { - newSublabel = null - } else { - const trimmed = rawSublabel.trim() - newSublabel = trimmed.length === 0 ? null : trimmed - } - if (newSublabel !== null) { - const takenBy = await resolveContainerByBarcode(database, newSublabel) - if (takenBy !== null && takenBy !== id) { - return c.json({ error: `Sublabel '${newSublabel}' is already in use` }, 400) + if (newBarcode !== undefined) { + if (containerType === 'micronix_tube' && newBarcode !== null) { + await tx.update(micronixTube).set({ barcode: newBarcode }).where(eq(micronixTube.id, id)) + } else if (containerType === 'cryovial_tube') { + await tx.update(cryovialTube).set({ barcode: newBarcode }).where(eq(cryovialTube.id, id)) } } - } - - // Update container fields (excluding tagIds and barcode on subtype) - const user = c.get('user') - const [updated] = await database - .update(storageContainer) - .set({ - ...updateData, - lastUpdated: utcNow(), - updatedBy: user?.id, - }) - .where(eq(storageContainer.id, id)) - .returning() - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- runtime invariant per avoid-masking-bugs: update must return row - if (!updated) { - return c.json({ error: 'Container not found' }, 404) - } - - // Update tags if provided - if (tagIds !== undefined) { - // Remove all existing tags - await database.delete(storageContainerTag) - .where(eq(storageContainerTag.storageContainerId, id)) - - // Add new tags - if (tagIds.length > 0) { - await database.insert(storageContainerTag).values( - tagIds.map(tagId => ({ - storageContainerId: id, - tagId, - })) - ) + if (newSublabel !== undefined) { + await tx.update(paper).set({ sublabel: newSublabel }).where(eq(paper.id, id)) } - } - - if (rawBarcode !== undefined) { - if (containerType === 'micronix_tube' && newBarcode != null) { - await database.update(micronixTube).set({ barcode: newBarcode }).where(eq(micronixTube.id, id)) - } else if (containerType === 'cryovial_tube' && newBarcode !== undefined) { - await database.update(cryovialTube).set({ barcode: newBarcode }).where(eq(cryovialTube.id, id)) - } - } - if (rawSublabel !== undefined && containerType === 'paper') { - await database.update(paper).set({ sublabel: newSublabel ?? null }).where(eq(paper.id, id)) - } + return row + }) - // Return enriched container const enriched = await enrichContainerForApi(database, updated) return c.json({ container: mapEnrichedContainerToWire(enriched) }) - } catch (error) { - if (error instanceof z.ZodError) { - return c.json({ error: 'Invalid input', details: error.issues }, 400) - } - return c.json({ error: 'Internal server error' }, 500) + } catch (error: unknown) { + return handleRouteError(error, c) } }) diff --git a/packages/web/src/components/ContainerEditModal.tsx b/packages/web/src/components/ContainerEditModal.tsx index d600d54f..eae190bb 100644 --- a/packages/web/src/components/ContainerEditModal.tsx +++ b/packages/web/src/components/ContainerEditModal.tsx @@ -3,6 +3,8 @@ import { tagsApi } from '../lib/api/reference-data'; import type { Tag } from '../lib/api/reference-data'; import { settingsApi } from '../lib/api/settings'; import type { Unit } from '../lib/api/types'; +import type { ContainerUpdate } from '../lib/api/containers' +import { useUpdateContainer } from '../hooks/useContainers' import { useHotkey, useModifierHotkey } from '../hooks/useHotkey' import { Modal } from '../ui' @@ -16,8 +18,10 @@ interface ContainerEditModalProps { tags?: Array<{ id: number; name: string }> unit?: { id: number; symbol: string } containerType?: string - /** Subtype barcode; editable for micronix_tube, cryovial_tube, paper (not static_well) */ + /** Tube barcode; editable for micronix_tube and cryovial_tube */ barcode?: string | null + /** Paper sublabel; editable for paper */ + sublabel?: string | null } onSuccess: () => void } @@ -25,9 +29,15 @@ interface ContainerEditModalProps { type ContainerForEdit = ContainerEditModalProps['container'] function canEditBarcode(containerType: string | undefined): boolean { - return ( - containerType === 'micronix_tube' || containerType === 'cryovial_tube' || containerType === 'paper' - ) + return containerType === 'micronix_tube' || containerType === 'cryovial_tube' +} + +function errorMessageOf(err: unknown): string { + if (err && typeof err === 'object' && 'response' in err) { + const message = (err as { response?: { data?: { error?: unknown } } }).response?.data?.error + if (typeof message === 'string' && message.length > 0) return message + } + return err instanceof Error && err.message ? err.message : 'Failed to update container' } function ContainerEditModalForm({ @@ -46,6 +56,7 @@ function ContainerEditModalForm({ const [loadingTags, setLoadingTags] = useState(false) const [loadingUnits, setLoadingUnits] = useState(false) const formRef = useRef(null) + const updateContainer = useUpdateContainer() const [formData, setFormData] = useState({ comment: container.comment || '', @@ -53,6 +64,7 @@ function ContainerEditModalForm({ unitId: container.unit?.id || undefined, tagIds: container.tags?.map(t => t.id) || [], barcode: container.barcode ?? '', + sublabel: container.sublabel ?? '', }) // Load tags and units when form mounts (key={container.id} resets form per container) @@ -95,13 +107,7 @@ function ContainerEditModalForm({ setError(null) try { - const updateData: { - comment?: string - remainingQuantity?: number - unitId?: number - tagIds?: number[] - barcode?: string | null - } = {} + const updateData: ContainerUpdate = {} // Only include fields that have changed if (formData.comment !== (container.comment || '')) { @@ -158,30 +164,27 @@ function ContainerEditModalForm({ } } + if (container.containerType === 'paper') { + const t = formData.sublabel.trim() + const next = t.length > 0 ? t : null + const previous = container.sublabel?.trim() || null + if (next !== previous) { + updateData.sublabel = next + } + } + // Only send request if there are changes if (Object.keys(updateData).length === 0) { onClose() return } - const response = await fetch(`/api/containers/${container.id}`, { - method: 'PATCH', - headers: { - 'Content-Type': 'application/json', - }, - credentials: 'include', - body: JSON.stringify(updateData), - }) - - if (!response.ok) { - const errorData = await response.json() - throw new Error(errorData.error || 'Failed to update container') - } + await updateContainer.mutateAsync({ id: container.id, data: updateData }) onSuccess() onClose() - } catch (err: any) { - setError(err.message || 'Failed to update container') + } catch (err: unknown) { + setError(errorMessageOf(err)) } finally { setLoading(false) } @@ -248,6 +251,25 @@ function ContainerEditModalForm({ )} + {container.containerType === 'paper' && ( +
+ + setFormData({ ...formData, sublabel: e.target.value })} + className="w-full px-3 py-2 border border-app-border rounded-md shadow-sm font-mono focus:ring-app-accent focus:border-app-accent" + disabled={loading} + autoComplete="off" + placeholder="Spot identifier on the sheet" + /> +

To move it to another sheet, use Move Papers in the app menu.

+
+ )} + {/* Unit */} {container.containerType && (
diff --git a/packages/web/src/components/__tests__/ContainerEditModal.test.tsx b/packages/web/src/components/__tests__/ContainerEditModal.test.tsx index 530c9a63..bf5ce7e9 100644 --- a/packages/web/src/components/__tests__/ContainerEditModal.test.tsx +++ b/packages/web/src/components/__tests__/ContainerEditModal.test.tsx @@ -1,7 +1,8 @@ -import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest' +import { describe, it, expect, vi, beforeEach } from 'vitest' import userEvent from '@testing-library/user-event' import { render, screen, waitFor } from '../../__tests__/helpers/render' import ContainerEditModal from '../ContainerEditModal' +import { containersApi } from '../../lib/api/containers' vi.mock('../../lib/api/reference-data', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -17,17 +18,17 @@ vi.mock('../../lib/api/settings', async () => { }) }) -describe('ContainerEditModal', () => { - const fetchMock = vi.fn() - - beforeEach(() => { - fetchMock.mockReset() - fetchMock.mockResolvedValue({ ok: true, json: async () => ({}) }) - vi.stubGlobal('fetch', fetchMock) +vi.mock('../../lib/api/containers', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('containers', { + containersApi: { list: vi.fn(), get: vi.fn(), update: vi.fn() }, }) +}) - afterEach(() => { - vi.unstubAllGlobals() +describe('ContainerEditModal', () => { + beforeEach(() => { + vi.mocked(containersApi.update).mockReset() + vi.mocked(containersApi.update).mockResolvedValue({ id: 7, containerType: 'static_well' } as never) }) it('saves a cleared comment', async () => { @@ -46,9 +47,97 @@ describe('ContainerEditModal', () => { await user.click(screen.getByRole('button', { name: 'Save Changes' })) await waitFor(() => expect(onSuccess).toHaveBeenCalled()) - expect(fetchMock).toHaveBeenCalledTimes(1) - const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit] - expect(url).toBe('/api/containers/7') - expect(JSON.parse(init.body as string)).toEqual({ comment: '' }) + expect(containersApi.update).toHaveBeenCalledTimes(1) + expect(containersApi.update).toHaveBeenCalledWith(7, { comment: '' }) + }) + + it('edits a Paper sublabel and sends it as the sublabel', async () => { + const user = userEvent.setup() + const onSuccess = vi.fn() + await render( + {}} + onSuccess={onSuccess} + container={{ id: 93926, containerType: 'paper', sublabel: '2067376' }} + />, + ) + + expect(screen.queryByLabelText(/Barcode/)).not.toBeInTheDocument() + const sublabel = screen.getByLabelText('Sublabel') + expect(sublabel).toHaveValue('2067376') + await user.clear(sublabel) + await user.type(sublabel, ' 2067376-B ') + await user.click(screen.getByRole('button', { name: 'Save Changes' })) + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(containersApi.update).toHaveBeenCalledWith(93926, { sublabel: '2067376-B' }) + }) + + it('clears a Paper sublabel by sending null', async () => { + const user = userEvent.setup() + const onSuccess = vi.fn() + await render( + {}} + onSuccess={onSuccess} + container={{ id: 93926, containerType: 'paper', sublabel: '2067376' }} + />, + ) + + await user.clear(screen.getByLabelText('Sublabel')) + await user.click(screen.getByRole('button', { name: 'Save Changes' })) + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(containersApi.update).toHaveBeenCalledWith(93926, { sublabel: null }) + }) + + it('shows the API error and stays open when the save fails', async () => { + const user = userEvent.setup() + const onSuccess = vi.fn() + const onClose = vi.fn() + vi.mocked(containersApi.update).mockRejectedValue( + Object.assign(new Error('Request failed with status code 400'), { + response: { status: 400, data: { error: "Sublabel '2067377' is already in use" } }, + }), + ) + await render( + , + ) + + await user.clear(screen.getByLabelText('Sublabel')) + await user.type(screen.getByLabelText('Sublabel'), '2067377') + await user.click(screen.getByRole('button', { name: 'Save Changes' })) + + expect(await screen.findByText("Sublabel '2067377' is already in use")).toBeInTheDocument() + expect(onSuccess).not.toHaveBeenCalled() + expect(onClose).not.toHaveBeenCalled() + }) + + it('edits a tube barcode and shows no Sublabel field', async () => { + const user = userEvent.setup() + const onSuccess = vi.fn() + await render( + {}} + onSuccess={onSuccess} + container={{ id: 8, containerType: 'cryovial_tube', barcode: 'CRYO-1' }} + />, + ) + + expect(screen.queryByLabelText('Sublabel')).not.toBeInTheDocument() + await user.clear(screen.getByLabelText('Barcode')) + await user.type(screen.getByLabelText('Barcode'), 'CRYO-2') + await user.click(screen.getByRole('button', { name: 'Save Changes' })) + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(containersApi.update).toHaveBeenCalledWith(8, { barcode: 'CRYO-2' }) }) }) diff --git a/packages/web/src/hooks/__tests__/useContainers.test.tsx b/packages/web/src/hooks/__tests__/useContainers.test.tsx new file mode 100644 index 00000000..c5329ed0 --- /dev/null +++ b/packages/web/src/hooks/__tests__/useContainers.test.tsx @@ -0,0 +1,52 @@ +import { describe, it, expect, vi } from 'vitest' +import { renderHook, waitFor } from '@testing-library/react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import type { ReactNode } from 'react' +import { containerKeys, useUpdateContainer } from '../useContainers' +import { containersApi } from '../../lib/api/containers' + +vi.mock('../../lib/api/containers', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('containers', { + containersApi: { list: vi.fn(), get: vi.fn(), update: vi.fn() }, + }) +}) + +function setup() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 }, mutations: { retry: false } }, + }) + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ) + return { queryClient, wrapper } +} + +describe('useUpdateContainer', () => { + it('saves the edit and invalidates every container query', async () => { + const updated = { id: 5, containerType: 'paper' as const, sublabel: 'NEW' } + vi.mocked(containersApi.update).mockResolvedValue(updated as never) + const { queryClient, wrapper } = setup() + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + + const { result } = renderHook(() => useUpdateContainer(), { wrapper }) + result.current.mutate({ id: 5, data: { sublabel: 'NEW' } }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(containersApi.update).toHaveBeenCalledWith(5, { sublabel: 'NEW' }) + expect(result.current.data).toEqual(updated) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: containerKeys.all }) + }) + + it('invalidates nothing when the save fails', async () => { + vi.mocked(containersApi.update).mockRejectedValue(new Error('Unknown tag id: 9')) + const { queryClient, wrapper } = setup() + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + + const { result } = renderHook(() => useUpdateContainer(), { wrapper }) + result.current.mutate({ id: 5, data: { tagIds: [9] } }) + + await waitFor(() => expect(result.current.isError).toBe(true)) + expect(invalidateSpy).not.toHaveBeenCalled() + }) +}) diff --git a/packages/web/src/hooks/useContainers.ts b/packages/web/src/hooks/useContainers.ts index 81dc9e60..669e3899 100644 --- a/packages/web/src/hooks/useContainers.ts +++ b/packages/web/src/hooks/useContainers.ts @@ -1,5 +1,5 @@ -import { useQuery } from '@tanstack/react-query' -import { containersApi, type ContainerDetail } from '../lib/api/containers' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' +import { containersApi, type ContainerDetail, type ContainerUpdate } from '../lib/api/containers' import { derivationsApi } from '../lib/api/derivations' export const containerKeys = { @@ -70,3 +70,16 @@ export function useContainerDerivationTree(containerId: number) { enabled: Number.isFinite(containerId) && containerId > 0, }) } + +/** + * Save a Container edit. Invalidates every container query: the edit changes the detail, and + * derivation trees and lists show the Container's quantity, Tags and identifier too. + * Shows no toast; the caller reports the error. + */ +export function useUpdateContainer() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: ({ id, data }: { id: number; data: ContainerUpdate }) => containersApi.update(id, data), + onSuccess: () => queryClient.invalidateQueries({ queryKey: containerKeys.all }), + }) +} diff --git a/packages/web/src/lib/api/__tests__/containers.test.ts b/packages/web/src/lib/api/__tests__/containers.test.ts index d47a6a83..1be75497 100644 --- a/packages/web/src/lib/api/__tests__/containers.test.ts +++ b/packages/web/src/lib/api/__tests__/containers.test.ts @@ -6,6 +6,7 @@ import { api } from '../client' vi.mock('../client', () => ({ api: { get: vi.fn(), + patch: vi.fn(), }, })) @@ -160,4 +161,28 @@ describe('containersApi', () => { }) }) }) + + describe('update', () => { + it('PATCHes the edit and resolves to the updated container', async () => { + const container = { + id: 93926, + specimenId: 1, + containerType: 'paper' as const, + sublabel: '2067376-B', + collection: { type: 'sheet' as const, id: 3, name: 'Sheet 1' }, + } + vi.mocked(api.patch).mockResolvedValue({ container }) + + const result = await containersApi.update(93926, { sublabel: '2067376-B' }) + + expect(api.patch).toHaveBeenCalledWith('/containers/93926', { sublabel: '2067376-B' }) + expect(result).toEqual(container) + }) + + it('throws ApiContractError when the body has no container', async () => { + vi.mocked(api.patch).mockResolvedValue({ ok: true }) + + await expect(containersApi.update(1, { comment: 'x' })).rejects.toBeInstanceOf(ApiContractError) + }) + }) }) diff --git a/packages/web/src/lib/api/containers.ts b/packages/web/src/lib/api/containers.ts index f62b673c..ecdfada9 100644 --- a/packages/web/src/lib/api/containers.ts +++ b/packages/web/src/lib/api/containers.ts @@ -2,6 +2,7 @@ import type { EnrichedContainerWire, SpecimenSummaryWire } from '@sampledb/contr import { api } from './client' import { parseContainerDetailWire, + parseContainerUpdate, parseContainersList, type ParsedContainerDetailWire, } from './parse-response' @@ -75,6 +76,19 @@ export type ContainersListParams = { limit?: number } +/** Fields PATCH /containers/:id accepts. Omit a field to leave it unchanged. */ +export type ContainerUpdate = { + comment?: string + remainingQuantity?: number + unitId?: number + /** Replaces the Container's Tags. */ + tagIds?: number[] + /** Micronix or cryovial tube barcode; null clears a cryovial barcode. */ + barcode?: string | null + /** Paper sublabel; null clears it. */ + sublabel?: string | null +} + export const containersApi = { list: async (params?: ContainersListParams) => { const body = await api.get('/containers', { params }) @@ -85,4 +99,9 @@ export const containersApi = { const body = await api.get(`/containers/${id}`) return normalizeContainerDetail(parseContainerDetailWire(body)) }, + + update: async (id: number, data: ContainerUpdate): Promise => { + const body = await api.patch(`/containers/${id}`, data) + return parseContainerUpdate(body) + }, } diff --git a/packages/web/src/lib/api/parse-response.ts b/packages/web/src/lib/api/parse-response.ts index 4650ad2f..52cc2db7 100644 --- a/packages/web/src/lib/api/parse-response.ts +++ b/packages/web/src/lib/api/parse-response.ts @@ -2,6 +2,7 @@ import { z } from 'zod' import { parseContainerDetailWire as parseContainerDetailWireSchema, parseContainersListWire, + enrichedContainerWireSchema, type ContainerDetailWire, type EnrichedContainerWire, } from '@sampledb/contract/wire' @@ -87,6 +88,13 @@ export function parseContainerDetailWire(body: unknown): ParsedContainerDetailWi } } +const containerUpdateWireSchema = z.object({ container: enrichedContainerWireSchema }) + +/** Parse the PATCH /containers/:id body and return the updated container. */ +export function parseContainerUpdate(body: unknown): EnrichedContainerWire { + return parseWithSchema(containerUpdateWireSchema, body, 'PATCH /containers/:id').container +} + export type ContainersListResult = { containers: EnrichedContainerWire[] pagination?: { page: number; limit: number; total: number; totalPages: number } diff --git a/packages/web/src/pages/ContainerDetail.tsx b/packages/web/src/pages/ContainerDetail.tsx index bbde5b05..f7d89763 100644 --- a/packages/web/src/pages/ContainerDetail.tsx +++ b/packages/web/src/pages/ContainerDetail.tsx @@ -705,12 +705,10 @@ export default function ContainerDetail() { tags: container.tags, unit: container.unit, containerType: effectiveContainerType, - barcode: tubeBarcode ?? paperSublabel, - }} - onSuccess={() => { - refreshContainer() - setShowEditModal(false) + barcode: tubeBarcode, + sublabel: paperSublabel, }} + onSuccess={() => setShowEditModal(false)} /> )} From 62b9fdabae342bfea1f7c5cf043f1b9839ef0ef0 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:32:58 -0700 Subject: [PATCH 02/54] fix(web): sort lists by declared type across the whole list (#116) - DataTable columns declare a sortType (text, number, date) instead of sortable; text sorts naturally (2 before 10) via a shared comparator in lib/sort-compare.ts, and missing values sort last in both directions. Strings are no longer sorted as dates because `new Date()` parses them. - Studies: compute ascending order for every key and negate only for descending, so Title and Lead ascending run A to Z. Date and count defaults (newest or largest first) are unchanged. - Studies loads every study (paging at the API's max page size) instead of infinite-scrolling pages of 50, so sorting and filtering cover the whole list. Sorting by a count prefetches every study's summary. Co-Authored-By: Claude Opus 5.5 --- .../content/docs/guides/workflows/studies.md | 2 +- .../__tests__/helpers/mock-api-templates.ts | 1 + packages/web/src/components/DataTable.tsx | 49 ++---- .../components/__tests__/DataTable.test.tsx | 90 ++++++++++- packages/web/src/hooks/README.md | 3 +- .../src/hooks/__tests__/useStudies.test.tsx | 6 +- packages/web/src/hooks/useStudies.ts | 47 +++--- .../src/lib/__tests__/sort-compare.test.ts | 48 ++++++ packages/web/src/lib/sort-compare.ts | 53 +++++++ packages/web/src/pages/BloodControls.tsx | 16 +- packages/web/src/pages/Collections.tsx | 6 +- .../web/src/pages/ControlDefinitionDetail.tsx | 6 +- packages/web/src/pages/QpcrExperiments.tsx | 20 ++- packages/web/src/pages/Specimens.tsx | 4 +- packages/web/src/pages/Studies.tsx | 149 +++++------------- packages/web/src/pages/StudyDetail.tsx | 8 +- .../web/src/pages/__tests__/Studies.test.tsx | 113 +++++++++++++ .../src/pages/__tests__/StudyDetail.test.tsx | 44 +++++- 18 files changed, 449 insertions(+), 216 deletions(-) create mode 100644 packages/web/src/lib/__tests__/sort-compare.test.ts create mode 100644 packages/web/src/lib/sort-compare.ts diff --git a/packages/docs/src/content/docs/guides/workflows/studies.md b/packages/docs/src/content/docs/guides/workflows/studies.md index 7da12d7f..50ebff82 100644 --- a/packages/docs/src/content/docs/guides/workflows/studies.md +++ b/packages/docs/src/content/docs/guides/workflows/studies.md @@ -52,7 +52,7 @@ Use the same short code in CSV files that you use in the UI. Avoid special chara ## Find studies -On the Studies page, search matches titles, short codes, and lead person names. You can filter and sort by creation date, title, and other attributes. +On the Studies page, search matches titles, short codes, and lead person names. You can filter by type and lead, and sort the whole list by last updated date, title, lead, or subject, specimen, and container counts. Text sorts read numbers in order, so `2` comes before `10`. ## Add subjects diff --git a/packages/web/src/__tests__/helpers/mock-api-templates.ts b/packages/web/src/__tests__/helpers/mock-api-templates.ts index dc1dd187..e2094a04 100644 --- a/packages/web/src/__tests__/helpers/mock-api-templates.ts +++ b/packages/web/src/__tests__/helpers/mock-api-templates.ts @@ -51,6 +51,7 @@ export function studiesPageMock(): MockApiOverrides { return { studiesApi: { list: vi.fn().mockResolvedValue(emptyStudiesList), + getSummaries: vi.fn().mockResolvedValue({ summaries: [] }), }, } } diff --git a/packages/web/src/components/DataTable.tsx b/packages/web/src/components/DataTable.tsx index 9d4f0521..99496e25 100644 --- a/packages/web/src/components/DataTable.tsx +++ b/packages/web/src/components/DataTable.tsx @@ -1,11 +1,13 @@ import { useState, useEffect, useRef } from 'react' import { SkeletonTable } from '../ui' import Pagination from './Pagination' +import { compareBySortType, type SortType } from '../lib/sort-compare' export interface Column { key: keyof T | string label: string - sortable?: boolean + /** How the column sorts: text (natural order), number, or date. Omit for a column that does not sort. */ + sortType?: SortType render?: (value: any, row: T) => React.ReactNode } @@ -48,7 +50,7 @@ export default function DataTable({ const prevResetDepsRef = useRef([data.length, sortColumn, sortDirection, pagination?.page ?? 0]) const handleSort = (column: Column) => { - if (!column.sortable) return + if (!column.sortType) return if (sortColumn === column.key) { setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc') @@ -58,38 +60,13 @@ export default function DataTable({ } } - const sortedData = [...data].sort((a, b) => { - if (!sortColumn) return 0 - - const aValue = a[sortColumn as keyof T] - const bValue = b[sortColumn as keyof T] - - if (aValue === bValue) return 0 - if (aValue === null || aValue === undefined) return 1 - if (bValue === null || bValue === undefined) return -1 - - // Handle numeric comparison for specimenCount and dates - if (typeof aValue === 'number' && typeof bValue === 'number') { - const comparison = aValue < bValue ? -1 : 1 - return sortDirection === 'asc' ? comparison : -comparison - } - - // Handle date strings - if (typeof aValue === 'string' && typeof bValue === 'string') { - const aDate = new Date(aValue).getTime() - const bDate = new Date(bValue).getTime() - if (!isNaN(aDate) && !isNaN(bDate)) { - const comparison = aDate < bDate ? -1 : 1 - return sortDirection === 'asc' ? comparison : -comparison - } - } - - // String comparison - const aStr = String(aValue).toLowerCase() - const bStr = String(bValue).toLowerCase() - const comparison = aStr < bStr ? -1 : 1 - return sortDirection === 'asc' ? comparison : -comparison - }) + const sortType = columns.find((column) => column.key === sortColumn)?.sortType + const sortedData = + sortColumn && sortType + ? [...data].sort((a, b) => + compareBySortType(a[sortColumn as keyof T], b[sortColumn as keyof T], sortType, sortDirection) + ) + : data // Apply pagination if enabled const paginatedData = pagination @@ -203,12 +180,12 @@ export default function DataTable({ key={String(column.key)} onClick={() => handleSort(column)} className={`${density === 'compact' ? 'px-3 py-2' : 'px-6 py-3'} text-left text-xs font-medium text-app-text-muted uppercase ${ - column.sortable ? 'cursor-pointer hover:bg-app-border/50' : '' + column.sortType ? 'cursor-pointer hover:bg-app-border/50' : '' }`} >
{column.label} - {column.sortable && sortColumn === column.key && ( + {column.sortType && sortColumn === column.key && ( { const columns: Column[] = [ - { key: 'id', label: 'ID', sortable: true }, - { key: 'name', label: 'Name', sortable: true }, + { key: 'id', label: 'ID', sortType: 'number' }, + { key: 'name', label: 'Name', sortType: 'text' }, { key: 'value', label: 'Value', render: (value) => `$${value.toFixed(2)}`, - sortable: true, + sortType: 'number', }, ] @@ -118,5 +118,85 @@ describe('DataTable', () => { expect(screen.getByText('ID')).toBeInTheDocument() expect(screen.getByText('Name')).toBeInTheDocument() }) -}) + describe('sorting by declared type', () => { + interface SubjectRow { + id: number + name: string + specimenCount: number | null + created: string | null + } + + const subjectColumns: Column[] = [ + { key: 'name', label: 'Name', sortType: 'text' }, + { key: 'specimenCount', label: 'Specimens', sortType: 'number' }, + { key: 'created', label: 'Created', sortType: 'date' }, + ] + + // Names like study 70's: numeric names (which `new Date()` can parse) and alphanumeric ones. + const subjects: SubjectRow[] = [ + { id: 1, name: '140', specimenCount: 3, created: '2024-03-01T00:00:00Z' }, + { id: 2, name: '100', specimenCount: 12, created: '2023-11-15T00:00:00Z' }, + { id: 3, name: '2', specimenCount: null, created: '2024-01-10T00:00:00Z' }, + { id: 4, name: '29PH', specimenCount: 1, created: null }, + { id: 5, name: '10', specimenCount: 100, created: '2022-06-30T00:00:00Z' }, + { id: 6, name: '1', specimenCount: 2, created: '2024-12-31T00:00:00Z' }, + { id: 7, name: '350', specimenCount: 20, created: '2023-01-01T00:00:00Z' }, + { id: 8, name: '3CDX', specimenCount: 5, created: '2021-05-05T00:00:00Z' }, + ] + + function columnValues(columnIndex: number): string[] { + const rows = screen.getAllByRole('row').slice(1) + return rows.map((row) => within(row).getAllByRole('cell')[columnIndex].textContent) + } + + it('sorts text naturally, so numeric names run 1, 2, 10, 100', async () => { + const user = userEvent.setup() + await render() + + await user.click(screen.getByRole('columnheader', { name: 'Name' })) + expect(columnValues(0)).toEqual(['1', '2', '3CDX', '10', '29PH', '100', '140', '350']) + + await user.click(screen.getByRole('columnheader', { name: 'Name' })) + expect(columnValues(0)).toEqual(['350', '140', '100', '29PH', '10', '3CDX', '2', '1']) + }) + + it('sorts a number column by value and puts missing values last', async () => { + const user = userEvent.setup() + await render() + + await user.click(screen.getByRole('columnheader', { name: 'Specimens' })) + expect(columnValues(1)).toEqual(['1', '2', '3', '5', '12', '20', '100', '']) + + await user.click(screen.getByRole('columnheader', { name: 'Specimens' })) + expect(columnValues(1)).toEqual(['100', '20', '12', '5', '3', '2', '1', '']) + }) + + it('sorts a date column by date and puts missing dates last', async () => { + const user = userEvent.setup() + await render() + + await user.click(screen.getByRole('columnheader', { name: 'Created' })) + expect(columnValues(0)).toEqual(['3CDX', '10', '350', '100', '2', '140', '1', '29PH']) + + await user.click(screen.getByRole('columnheader', { name: 'Created' })) + expect(columnValues(0)).toEqual(['1', '140', '2', '100', '350', '10', '3CDX', '29PH']) + }) + + it('starts sorted by initialSortColumn in initialSortDirection', async () => { + await render( + + ) + expect(columnValues(0).slice(0, 3)).toEqual(['1', '2', '3CDX']) + }) + + it('does not sort when a column declares no sort type', async () => { + const user = userEvent.setup() + const unsortable: Column[] = [{ key: 'name', label: 'Name' }] + await render() + + await user.click(screen.getByRole('columnheader', { name: 'Name' })) + expect(columnValues(0)).toEqual(subjects.map((s) => s.name)) + }) + }) +}) diff --git a/packages/web/src/hooks/README.md b/packages/web/src/hooks/README.md index 08cea6ac..c3459256 100644 --- a/packages/web/src/hooks/README.md +++ b/packages/web/src/hooks/README.md @@ -23,8 +23,7 @@ After a **write**, ask: **Study create** (`useCreateStudy`): ```typescript -queryClient.invalidateQueries({ queryKey: studyKeys.lists() }) -queryClient.invalidateQueries({ queryKey: studyKeys.infinite() }) +queryClient.invalidateQueries({ queryKey: studyKeys.all }) invalidateDashboardQueries(queryClient) ``` diff --git a/packages/web/src/hooks/__tests__/useStudies.test.tsx b/packages/web/src/hooks/__tests__/useStudies.test.tsx index f6fc6b0c..10772eab 100644 --- a/packages/web/src/hooks/__tests__/useStudies.test.tsx +++ b/packages/web/src/hooks/__tests__/useStudies.test.tsx @@ -202,12 +202,12 @@ describe('useStudies Hooks', () => { expect(invalidateSpy).toHaveBeenCalled() }) - it('invalidates the Studies page infinite list (limit 50)', async () => { + it('invalidates the Studies page list of all studies', async () => { vi.mocked(studiesApi.create).mockResolvedValue({ study: { id: 4, title: 'T', shortCode: 'T4', isLongitudinal: false, leadPerson: 'L', created: '2024-01-01', lastUpdated: '2024-01-01' }, }) const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false }, mutations: { retry: false } } }) - queryClient.setQueryData(studyKeys.infinite(50), { pages: [], pageParams: [] }) + queryClient.setQueryData(studyKeys.browse(''), { studies: [], total: 0 }) const wrapper = ({ children }: { children: ReactNode }) => ( {children} @@ -218,7 +218,7 @@ describe('useStudies Hooks', () => { result.current.mutate({ title: 'T', shortCode: 'T4', isLongitudinal: false, leadPerson: 'L' }) await waitFor(() => expect(result.current.isSuccess).toBe(true)) - expect(queryClient.getQueryState(studyKeys.infinite(50))?.isInvalidated).toBe(true) + expect(queryClient.getQueryState(studyKeys.browse(''))?.isInvalidated).toBe(true) }) }) diff --git a/packages/web/src/hooks/useStudies.ts b/packages/web/src/hooks/useStudies.ts index ed9511eb..2d9f8527 100644 --- a/packages/web/src/hooks/useStudies.ts +++ b/packages/web/src/hooks/useStudies.ts @@ -1,7 +1,6 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useQuery, - useInfiniteQuery, useMutation, useQueryClient, type QueryClient, @@ -16,7 +15,8 @@ export const studyKeys = { lists: () => [...studyKeys.all, 'list'] as const, list: (filters?: { search?: string; page?: number; limit?: number }) => [...studyKeys.lists(), filters] as const, - infinite: (limit?: number) => [...studyKeys.all, 'infinite', { limit }] as const, + /** Every study matching the search, for the Studies page. */ + browse: (search: string) => [...studyKeys.lists(), 'browse', { search }] as const, leadPersons: () => [...studyKeys.all, 'lead-persons'] as const, details: () => [...studyKeys.all, 'detail'] as const, detail: (id: number) => [...studyKeys.details(), id] as const, @@ -24,7 +24,8 @@ export const studyKeys = { summaryCard: (id: number) => [...studyKeys.all, 'summary-card', id] as const, } -const STUDIES_PAGE_LIMIT = 50 +/** Page size for loading every study; the API clamps it to its configured maximum. */ +const ALL_STUDIES_PAGE_LIMIT = 1000 const SUMMARY_CARD_STALE_MS = 5 * 60 * 1000 function summaryBatchKey(ids: number[]) { @@ -116,26 +117,28 @@ export function useStudies(search?: string, params?: { page?: number; limit?: nu }) } -/** Unfiltered browse with server pagination (infinite scroll). */ -export function useInfiniteStudies(limit = STUDIES_PAGE_LIMIT) { - return useInfiniteQuery({ - queryKey: studyKeys.infinite(limit), - queryFn: async ({ pageParam }) => studiesApi.list(undefined, { page: pageParam, limit }), - initialPageParam: 1, - getNextPageParam: (lastPage, _pages, lastPageParam) => { - const pagination = lastPage.pagination - if (!pagination || lastPageParam >= pagination.totalPages) return undefined - return lastPageParam + 1 - }, - }) +/** + * Every study matching the search, fetching each page the API reports, so the page can + * sort and filter the whole list rather than the pages loaded so far. + */ +export async function listAllStudies(search?: string): Promise<{ studies: Study[]; total: number }> { + const first = await studiesApi.list(search || undefined, { page: 1, limit: ALL_STUDIES_PAGE_LIMIT }) + const totalPages = first.pagination?.totalPages ?? 1 + const rest = await Promise.all( + Array.from({ length: Math.max(0, totalPages - 1) }, (_, i) => + studiesApi.list(search || undefined, { page: i + 2, limit: ALL_STUDIES_PAGE_LIMIT }) + ) + ) + const studies = [...first.studies, ...rest.flatMap((page) => page.studies)] + return { studies, total: first.pagination?.total ?? studies.length } } -/** All studies matching filters (client-side sort/pagination on the list page). */ -export function useStudiesFiltered(search: string, enabled: boolean) { +/** All studies matching the search (the Studies page sorts and filters them client-side). */ +export function useAllStudies(search: string, options?: { enabled?: boolean }) { return useQuery({ - queryKey: studyKeys.list({ search, page: 1, limit: 10000 }), - queryFn: () => studiesApi.list(search || undefined, { page: 1, limit: 10000 }), - enabled, + queryKey: studyKeys.browse(search), + queryFn: () => listAllStudies(search), + enabled: options?.enabled ?? true, }) } @@ -143,7 +146,7 @@ export function useStudyLeadPersons() { return useQuery({ queryKey: studyKeys.leadPersons(), queryFn: async () => { - const res = await studiesApi.list(undefined, { page: 1, limit: 10000 }) + const res = await listAllStudies() const leads = new Set(res.studies.map((s) => s.leadPerson).filter(Boolean)) return Array.from(leads).sort() as string[] }, @@ -219,7 +222,7 @@ export function useCreateStudy(options?: { silent?: boolean }) { showError(message || 'Failed to create study') }, onSuccess: () => { - // Every studies query (lists, infinite pages, lead persons, summaries) can change. + // Every studies query (lists, lead persons, summaries) can change. void queryClient.invalidateQueries({ queryKey: studyKeys.all }) void queryClient.invalidateQueries({ queryKey: dashboardKeys.all }) success('Study created successfully') diff --git a/packages/web/src/lib/__tests__/sort-compare.test.ts b/packages/web/src/lib/__tests__/sort-compare.test.ts new file mode 100644 index 00000000..a897e639 --- /dev/null +++ b/packages/web/src/lib/__tests__/sort-compare.test.ts @@ -0,0 +1,48 @@ +import { describe, it, expect } from 'vitest' +import { compareBySortType, compareText, type SortDirection, type SortType } from '../sort-compare' + +function sorted(values: unknown[], type: SortType, direction: SortDirection = 'asc'): unknown[] { + return [...values].sort((a, b) => compareBySortType(a, b, type, direction)) +} + +describe('compareText', () => { + it('orders digit runs by value and ignores case', () => { + const names = ['S10', 's2', 'S1', 'b', 'A'] + expect([...names].sort(compareText)).toEqual(['A', 'b', 'S1', 's2', 'S10']) + }) + + it('ignores leading and trailing whitespace', () => { + expect([' Dire Dawa', 'Accra', 'Zumba '].sort(compareText)).toEqual(['Accra', ' Dire Dawa', 'Zumba ']) + }) +}) + +describe('compareBySortType', () => { + it('sorts text naturally rather than as dates', () => { + expect(sorted(['100', '350', '10', '2', '1'], 'text')).toEqual(['1', '2', '10', '100', '350']) + expect(sorted(['100', '350', '10', '2', '1'], 'text', 'desc')).toEqual(['350', '100', '10', '2', '1']) + }) + + it('sorts numbers by value, including numeric strings', () => { + expect(sorted([10, '9', 100, 0.5], 'number')).toEqual([0.5, '9', 10, 100]) + }) + + it('sorts dates chronologically', () => { + const dates = ['2024-03-01', '2023-12-31T23:00:00Z', '2024-01-15'] + expect(sorted(dates, 'date')).toEqual(['2023-12-31T23:00:00Z', '2024-01-15', '2024-03-01']) + expect(sorted(dates, 'date', 'desc')).toEqual(['2024-03-01', '2024-01-15', '2023-12-31T23:00:00Z']) + }) + + it('puts missing and unparseable values last in both directions', () => { + expect(sorted([null, 3, '', 1], 'number')).toEqual([1, 3, null, '']) + expect(compareBySortType(undefined, 1, 'number', 'desc')).toBeGreaterThan(0) + expect(compareBySortType(1, undefined, 'number', 'asc')).toBeLessThan(0) + expect(sorted([null, 3, '', 1], 'number', 'desc')).toEqual([3, 1, null, '']) + expect(sorted(['not a date', '2024-01-01', null, '2023-01-01'], 'date', 'desc')).toEqual([ + '2024-01-01', + '2023-01-01', + 'not a date', + null, + ]) + expect(sorted([null, 'b', 'a'], 'text', 'desc')).toEqual(['b', 'a', null]) + }) +}) diff --git a/packages/web/src/lib/sort-compare.ts b/packages/web/src/lib/sort-compare.ts new file mode 100644 index 00000000..1ca3ac19 --- /dev/null +++ b/packages/web/src/lib/sort-compare.ts @@ -0,0 +1,53 @@ +/** How a sortable value is ordered. */ +export type SortType = 'text' | 'number' | 'date' + +export type SortDirection = 'asc' | 'desc' + +const naturalCollator = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }) + +/** + * Natural, case-insensitive text order that ignores surrounding whitespace: runs of + * digits compare by value, so `2` sorts before `10` and `S2` before `S10`. + */ +export function compareText(a: string, b: string): number { + return naturalCollator.compare(a.trim(), b.trim()) +} + +/** The comparable key for a value, or null when the value is missing or not of the type. */ +function sortKey(value: unknown, type: SortType): string | number | null { + if (value === null || value === undefined) return null + switch (type) { + case 'text': + return String(value) + case 'number': { + if (typeof value === 'string' && value.trim() === '') return null + const n = typeof value === 'number' ? value : Number(value) + return Number.isFinite(n) ? n : null + } + case 'date': { + if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value.getTime() + if (typeof value !== 'string' && typeof value !== 'number') return null + if (typeof value === 'string' && value.trim() === '') return null + const t = new Date(value).getTime() + return Number.isNaN(t) ? null : t + } + } +} + +/** + * Compare two values by their declared sort type in the given direction. Missing values + * (null, undefined, blank, or unparseable for numbers and dates) sort last in either + * direction. + */ +export function compareBySortType(a: unknown, b: unknown, type: SortType, direction: SortDirection): number { + const aKey = sortKey(a, type) + const bKey = sortKey(b, type) + if (aKey === null && bKey === null) return 0 + if (aKey === null) return 1 + if (bKey === null) return -1 + const ascending = + typeof aKey === 'string' && typeof bKey === 'string' + ? compareText(aKey, bKey) + : (aKey as number) - (bKey as number) + return direction === 'asc' ? ascending : -ascending +} diff --git a/packages/web/src/pages/BloodControls.tsx b/packages/web/src/pages/BloodControls.tsx index 6638d064..0bc24ac1 100644 --- a/packages/web/src/pages/BloodControls.tsx +++ b/packages/web/src/pages/BloodControls.tsx @@ -140,7 +140,7 @@ export default function BloodControls() { { key: 'name', label: 'Name', - sortable: true, + sortType: 'text', render: (value, row) => ( {value} @@ -200,7 +200,7 @@ export default function BloodControls() { { key: 'inventoryTotal', label: 'Inventory', - sortable: true, + sortType: 'number', render: (_, row) => { const hasSpots = (row.spotCount ?? 0) > 0 const hasMicronix = (row.micronixCount ?? 0) > 0 @@ -235,7 +235,7 @@ export default function BloodControls() { { key: 'targetDensity', label: 'Target Density', - sortable: true, + sortType: 'number', render: (value, row) => ( value ? (
@@ -345,7 +345,7 @@ export default function BloodControls() { { key: 'name', label: 'Batch Name', - sortable: true, + sortType: 'text', render: (value, row) => ( {value} @@ -355,7 +355,7 @@ export default function BloodControls() { { key: 'targetDensity', label: 'Target Density', - sortable: true, + sortType: 'number', render: (value, row) => ( value ? (
@@ -418,7 +418,7 @@ export default function BloodControls() { { key: 'inventoryTotal', label: 'Inventory', - sortable: true, + sortType: 'number', render: (_, row) => { const hasSpots = (row.spotCount ?? 0) > 0 const hasMicronix = (row.micronixCount ?? 0) > 0 @@ -446,7 +446,7 @@ export default function BloodControls() { { key: 'productionDate', label: 'Production Date', - sortable: true, + sortType: 'date', render: (value) => value ? ( {formatDate(value)} ) : Not set, @@ -690,8 +690,6 @@ export default function BloodControls() { data={compositionGroups} columns={compositionColumns} loading={overviewQuery.isPending} - initialSortColumn="strains" - initialSortDirection="asc" onRowClick={(row) => navigate(`/blood-controls/compositions/${encodeURIComponent(row.compositionKey)}`)} emptyMessage="No compositions match the current filters. Create a control definition to get started." pagination={{ diff --git a/packages/web/src/pages/Collections.tsx b/packages/web/src/pages/Collections.tsx index b00ff6c5..91fc284a 100644 --- a/packages/web/src/pages/Collections.tsx +++ b/packages/web/src/pages/Collections.tsx @@ -89,7 +89,7 @@ export default function Collections() { { key: 'name', label: 'Name', - sortable: true, + sortType: 'text', render: (_, row) => ( {row.name} @@ -99,7 +99,7 @@ export default function Collections() { { key: 'type', label: 'Type', - sortable: true, + sortType: 'text', render: (_, row) => getTypeLabel(row.type), }, { @@ -115,7 +115,7 @@ export default function Collections() { { key: 'itemCount', label: 'Items', - sortable: true, + sortType: 'number', render: (value) => (typeof value === 'number' ? value : '—'), }, ] diff --git a/packages/web/src/pages/ControlDefinitionDetail.tsx b/packages/web/src/pages/ControlDefinitionDetail.tsx index 8400a891..ec874191 100644 --- a/packages/web/src/pages/ControlDefinitionDetail.tsx +++ b/packages/web/src/pages/ControlDefinitionDetail.tsx @@ -61,7 +61,7 @@ export default function ControlDefinitionDetail() { { key: 'name', label: 'Batch Name', - sortable: true, + sortType: 'text', render: (value, row) => ( {value} @@ -71,7 +71,7 @@ export default function ControlDefinitionDetail() { { key: 'productionDate', label: 'Production Date', - sortable: true, + sortType: 'date', render: (value) => value ? ( {formatDate(value)} ) : Not set, @@ -79,7 +79,7 @@ export default function ControlDefinitionDetail() { { key: 'inventoryTotal', label: 'Inventory', - sortable: true, + sortType: 'number', render: (_, row) => { const hasSpots = (row.spotCount ?? 0) > 0 const hasMicronix = (row.micronixCount ?? 0) > 0 diff --git a/packages/web/src/pages/QpcrExperiments.tsx b/packages/web/src/pages/QpcrExperiments.tsx index ca2429a8..54df58d1 100644 --- a/packages/web/src/pages/QpcrExperiments.tsx +++ b/packages/web/src/pages/QpcrExperiments.tsx @@ -77,7 +77,7 @@ export default function QpcrExperiments() { { key: 'name', label: 'Name', - sortable: true, + sortType: 'text', render: (_, row) => ( {row.name?.trim() || `Experiment ${row.id}`} @@ -87,7 +87,7 @@ export default function QpcrExperiments() { { key: 'status', label: 'Status', - sortable: true, + sortType: 'text', render: (_, row) => ( {getStatusLabel(row.status)} @@ -97,7 +97,7 @@ export default function QpcrExperiments() { { key: 'templateFormat', label: 'Template', - sortable: true, + sortType: 'text', render: (_, row) => ( {(TEMPLATE_LABELS as Record)[row.templateFormat] ?? row.templateFormat} @@ -107,7 +107,7 @@ export default function QpcrExperiments() { { key: 'plateBarcode', label: 'Plate', - sortable: true, + sortType: 'text', render: (val) => ( {typeof val === 'string' ? val.trim() || '—' : '—'} ), @@ -115,7 +115,6 @@ export default function QpcrExperiments() { { key: 'targets', label: 'Target(s)', - sortable: false, render: (_, row) => { const targets = row.targets ?? [] const names = targets.map((t) => (t.targetName != null ? t.targetName.trim() : '')).filter(Boolean) // eslint-disable-line @typescript-eslint/no-unnecessary-condition -- targetName may be omitted @@ -127,7 +126,6 @@ export default function QpcrExperiments() { { key: 'assay', label: 'Assay', - sortable: false, render: (_, row) => { const targets = row.targets ?? [] if (targets.length === 0) return — @@ -144,7 +142,7 @@ export default function QpcrExperiments() { { key: 'wellCount', label: 'Wells', - sortable: true, + sortType: 'number', render: (_, row) => row.wellCount != null ? ( {row.wellCount} @@ -155,7 +153,7 @@ export default function QpcrExperiments() { { key: 'runCount', label: 'Runs', - sortable: true, + sortType: 'number', render: (_, row) => row.runCount != null ? ( {row.runCount} @@ -166,7 +164,7 @@ export default function QpcrExperiments() { { key: 'lastRunAt', label: 'Last run', - sortable: true, + sortType: 'date', render: (_, row) => ( {formatShortDate(row.lastRunAt ?? undefined)} ), @@ -174,7 +172,7 @@ export default function QpcrExperiments() { { key: 'created', label: 'Created', - sortable: true, + sortType: 'date', render: (val) => ( {formatShortDate(val as string | null)} ), @@ -182,7 +180,7 @@ export default function QpcrExperiments() { { key: 'lastUpdated', label: 'Updated', - sortable: true, + sortType: 'date', render: (val) => ( {formatShortDate(val as string | null)} ), diff --git a/packages/web/src/pages/Specimens.tsx b/packages/web/src/pages/Specimens.tsx index ebf44f8f..03b4ba08 100644 --- a/packages/web/src/pages/Specimens.tsx +++ b/packages/web/src/pages/Specimens.tsx @@ -113,13 +113,13 @@ export default function Specimens() { { key: 'collectionDate', label: 'Collected', - sortable: true, + sortType: 'date', render: (value) => formatTableDate(value as string), }, { key: 'created', label: 'Created', - sortable: true, + sortType: 'date', render: (value) => formatTableDate(value as string), }, ] diff --git a/packages/web/src/pages/Studies.tsx b/packages/web/src/pages/Studies.tsx index adb66246..af96a33b 100644 --- a/packages/web/src/pages/Studies.tsx +++ b/packages/web/src/pages/Studies.tsx @@ -1,16 +1,12 @@ -import { useEffect, useState, useRef, useMemo, useCallback, useEffectEvent } from 'react' +import { useEffect, useState, useRef, useMemo, useCallback } from 'react' import { Link, useSearchParams } from 'react-router-dom' import type { Study, StudySummaryBasic } from '../lib/api/studies' import StudyCard from '../components/StudyCard' import { useUser } from '../contexts/UserContext' import { useFocusSearchOnSlash } from '../hooks/useHotkey' import { useDebouncedValue } from '../hooks/useDebouncedValue' -import { - useInfiniteStudies, - useStudiesFiltered, - useStudyLeadPersons, - useStudySummaryCards, -} from '../hooks/useStudies' +import { useAllStudies, useStudyLeadPersons, useStudySummaryCards } from '../hooks/useStudies' +import { compareText, type SortDirection } from '../lib/sort-compare' import { AsyncPresentation, EmptyState, @@ -31,17 +27,31 @@ interface StudyWithSummary extends Study { summaryLoading?: boolean } -const PAGE_LIMIT = 50 - const SORT_OPTIONS: SortOption[] = ['title', 'date', 'subjects', 'specimens', 'containers', 'lead'] -type SortDirection = 'asc' | 'desc' - /** Names sort A-Z by default; dates and counts sort largest first. */ function defaultSortDirection(sortBy: SortOption): SortDirection { return sortBy === 'title' || sortBy === 'lead' ? 'asc' : 'desc' } +/** Ascending order for a sort key: A to Z for text, oldest or smallest first otherwise. */ +function compareStudiesAscending(a: StudyWithSummary, b: StudyWithSummary, sortBy: SortOption): number { + switch (sortBy) { + case 'title': + return compareText(a.title, b.title) + case 'lead': + return compareText(a.leadPerson, b.leadPerson) + case 'date': + return new Date(a.lastUpdated).getTime() - new Date(b.lastUpdated).getTime() + case 'subjects': + return (a.summary?.totalSubjects || 0) - (b.summary?.totalSubjects || 0) + case 'specimens': + return (a.summary?.totalSpecimens || 0) - (b.summary?.totalSpecimens || 0) + case 'containers': + return (a.summary?.totalContainers || 0) - (b.summary?.totalContainers || 0) + } +} + export default function Studies() { const [searchParams, setSearchParams] = useSearchParams() const { canWrite } = useUser() @@ -87,7 +97,6 @@ export default function Studies() { }) const { prefetch: prefetchSummaries, getCardState } = useStudySummaryCards() const summaryObserverRef = useRef(null) - const loadMoreRef = useRef(null) const cardRefs = useRef>(new Map()) const searchInputRef = useRef(null) useFocusSearchOnSlash(searchInputRef) @@ -98,18 +107,15 @@ export default function Studies() { [debouncedSearch, filterType, filterLead] ) - const infiniteQuery = useInfiniteStudies(PAGE_LIMIT) - const filteredQuery = useStudiesFiltered(debouncedSearch, hasActiveFilters) + // Load every study rather than pages of 50, so sorting and filtering cover the whole + // list. The search query narrows by title and short code on the server. + const allStudiesQuery = useAllStudies('') + const searchQuery = useAllStudies(debouncedSearch, { enabled: debouncedSearch !== '' }) const leadPersonsQuery = useStudyLeadPersons() - const activeQuery = hasActiveFilters ? filteredQuery : infiniteQuery + const activeQuery = debouncedSearch !== '' ? searchQuery : allStudiesQuery - const baseStudies = useMemo(() => { - if (hasActiveFilters) { - return filteredQuery.data?.studies ?? [] - } - return infiniteQuery.data?.pages.flatMap((page) => page.studies) ?? [] - }, [hasActiveFilters, filteredQuery.data, infiniteQuery.data]) + const baseStudies = useMemo(() => activeQuery.data?.studies ?? [], [activeQuery.data]) const studies: StudyWithSummary[] = useMemo( () => @@ -124,9 +130,7 @@ export default function Studies() { [baseStudies, getCardState] ) - const total = hasActiveFilters - ? (filteredQuery.data?.pagination?.total ?? studies.length) - : (infiniteQuery.data?.pages[0]?.pagination?.total ?? 0) + const total = allStudiesQuery.data?.total ?? 0 const listStatus = fromQuery(activeQuery, { isEmpty: activeQuery.isSuccess && baseStudies.length === 0, @@ -166,30 +170,14 @@ export default function Studies() { } }, [studies, prefetchSummaries]) - const loadNextPage = useEffectEvent(() => { - if (infiniteQuery.hasNextPage && !infiniteQuery.isFetchingNextPage) { - void infiniteQuery.fetchNextPage() - } - }) - + // Subject, specimen, and container counts come from per-study summaries that load as + // cards scroll into view. Sorting by a count needs every study's summary. + const sortsByCount = sortBy === 'subjects' || sortBy === 'specimens' || sortBy === 'containers' useEffect(() => { - if (hasActiveFilters || !infiniteQuery.hasNextPage) return - - const observer = new IntersectionObserver( - (entries) => { - if (entries.some((entry) => entry.isIntersecting)) loadNextPage() - }, - { rootMargin: '200px' } - ) - - if (loadMoreRef.current) { - observer.observe(loadMoreRef.current) + if (sortsByCount && baseStudies.length > 0) { + prefetchSummaries(baseStudies.map((study) => study.id)) } - - return () => observer.disconnect() - // isFetchingNextPage re-creates the observer after each page, which fires again - // while the sentinel is still visible, so short pages keep loading. - }, [hasActiveFilters, infiniteQuery.hasNextPage, infiniteQuery.isFetchingNextPage]) + }, [sortsByCount, baseStudies, prefetchSummaries]) const filteredAndSortedStudies = useMemo(() => { let filtered = [...studies] @@ -216,30 +204,8 @@ export default function Studies() { } filtered.sort((a, b) => { - let result = 0 - switch (sortBy) { - case 'title': - result = a.title.localeCompare(b.title) - break - case 'date': - result = new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime() - break - case 'lead': - result = a.leadPerson.localeCompare(b.leadPerson) - break - case 'subjects': - result = (b.summary?.totalSubjects || 0) - (a.summary?.totalSubjects || 0) - break - case 'specimens': - result = (b.summary?.totalSpecimens || 0) - (a.summary?.totalSpecimens || 0) - break - case 'containers': - result = (b.summary?.totalContainers || 0) - (a.summary?.totalContainers || 0) - break - default: - return 0 - } - return sortDirection === 'asc' ? -result : result + const ascending = compareStudiesAscending(a, b, sortBy) + return sortDirection === 'asc' ? ascending : -ascending }) return filtered @@ -558,49 +524,6 @@ export default function Studies() { ))}
- {!hasActiveFilters && ( - <> -
- {infiniteQuery.isFetchingNextPage && ( -
-
- - - - - Loading more studies... -
-
- )} - {!infiniteQuery.hasNextPage && studies.length > 0 && ( -
- No more studies to load -
- )} - - )}
diff --git a/packages/web/src/pages/StudyDetail.tsx b/packages/web/src/pages/StudyDetail.tsx index 3cf496f0..9f89f806 100644 --- a/packages/web/src/pages/StudyDetail.tsx +++ b/packages/web/src/pages/StudyDetail.tsx @@ -240,23 +240,23 @@ export default function StudyDetail() { } const subjectColumns: Column[] = [ - { key: 'name', label: 'Name', sortable: true }, + { key: 'name', label: 'Name', sortType: 'text' }, { key: 'specimenCount', label: 'Specimens', - sortable: true, + sortType: 'number', render: (value) => (value ?? 0).toLocaleString(), }, { key: 'created', label: 'Created', - sortable: true, + sortType: 'date', render: (value) => new Date(value as string).toLocaleDateString(), }, { key: 'lastUpdated', label: 'Last Updated', - sortable: true, + sortType: 'date', render: (value) => new Date(value as string).toLocaleDateString(), }, ] diff --git a/packages/web/src/pages/__tests__/Studies.test.tsx b/packages/web/src/pages/__tests__/Studies.test.tsx index cbb3f427..4f98da1b 100644 --- a/packages/web/src/pages/__tests__/Studies.test.tsx +++ b/packages/web/src/pages/__tests__/Studies.test.tsx @@ -1,7 +1,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { useNavigate } from 'react-router-dom' import userEvent from '@testing-library/user-event' +import { QueryClient } from '@tanstack/react-query' import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { studiesApi, type Study, type StudySummaryBasic } from '../../lib/api/studies' vi.mock('../../lib/api/studies', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -85,4 +87,115 @@ describe('Studies', () => { await user.click(screen.getByRole('button', { name: 'Back' })) await waitFor(() => expect(typeFilter).toHaveValue('all')) }) + + describe('sorting', () => { + function study(id: number, title: string, leadPerson: string, lastUpdated: string): Study { + return { + id, + title, + shortCode: `S${id}`, + isLongitudinal: false, + leadPerson, + created: '2020-01-01T00:00:00Z', + lastUpdated, + } + } + + const studies: Study[] = [ + study(1, 'gamma 10', 'Ng', '2024-02-01T00:00:00Z'), + study(2, 'Alpha', 'zhou', '2023-05-01T00:00:00Z'), + study(3, 'Gamma 2', 'Adeyemi', '2025-01-01T00:00:00Z'), + study(4, 'beta', 'Kim', '2022-09-01T00:00:00Z'), + ] + + function summary(studyId: number, totalSubjects: number): StudySummaryBasic { + return { studyId, totalSubjects, totalSpecimens: 0, totalContainers: 0, collectionDateRange: null } + } + + function listOnePage(list: Study[]) { + vi.mocked(studiesApi.list).mockResolvedValue({ + studies: list, + pagination: { page: 1, limit: 1000, total: list.length, totalPages: 1 }, + }) + } + + async function shownTitles(): Promise { + await screen.findAllByRole('heading', { level: 3 }) + return screen.getAllByRole('heading', { level: 3 }).map((h) => h.textContent) + } + + it('sorts Title ascending A to Z and descending Z to A', async () => { + listOnePage(studies) + const user = userEvent.setup() + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=title'] }) + + await waitFor(async () => expect(await shownTitles()).toEqual(['Alpha', 'beta', 'Gamma 2', 'gamma 10'])) + + await user.click(screen.getByRole('button', { name: 'Ascending' })) + await waitFor(async () => expect(await shownTitles()).toEqual(['gamma 10', 'Gamma 2', 'beta', 'Alpha'])) + expect(screen.getByRole('button', { name: 'Descending' })).toBeInTheDocument() + }) + + it('sorts Lead ascending A to Z and descending Z to A', async () => { + listOnePage(studies) + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=lead'] }) + // Leads: Adeyemi (Gamma 2), Kim (beta), Ng (gamma 10), zhou (Alpha) + await waitFor(async () => expect(await shownTitles()).toEqual(['Gamma 2', 'beta', 'gamma 10', 'Alpha'])) + }) + + it('sorts Lead descending Z to A', async () => { + listOnePage(studies) + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=lead&dir=desc'] }) + await waitFor(async () => expect(await shownTitles()).toEqual(['Alpha', 'gamma 10', 'beta', 'Gamma 2'])) + }) + + it('keeps date sort newest first by default and oldest first ascending', async () => { + listOnePage(studies) + const user = userEvent.setup() + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies'] }) + + await waitFor(async () => expect(await shownTitles()).toEqual(['Gamma 2', 'gamma 10', 'Alpha', 'beta'])) + + await user.click(screen.getByRole('button', { name: 'Descending' })) + await waitFor(async () => expect(await shownTitles()).toEqual(['beta', 'Alpha', 'gamma 10', 'Gamma 2'])) + }) + + it('sorts by a count over every study, largest first by default', async () => { + listOnePage(studies) + vi.mocked(studiesApi.getSummaries).mockResolvedValue({ + summaries: [summary(1, 5), summary(2, 300), summary(3, 40), summary(4, 0)], + }) + // Summary cards live in the query cache without observers, so keep them past render. + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=subjects'], queryClient }) + + // Summaries load for every study, not only the cards in view. + await waitFor(() => expect(studiesApi.getSummaries).toHaveBeenCalledWith(expect.arrayContaining([1, 2, 3, 4]))) + await waitFor(async () => expect(await shownTitles()).toEqual(['Alpha', 'Gamma 2', 'gamma 10', 'beta'])) + }) + + it('sorts the whole list, including studies past the first page', async () => { + vi.mocked(studiesApi.list).mockImplementation((_search, params) => + Promise.resolve( + params?.page === 2 + ? { studies: [studies[1]], pagination: { page: 2, limit: 3, total: 4, totalPages: 2 } } + : { + studies: [studies[0], studies[2], studies[3]], + pagination: { page: 1, limit: 3, total: 4, totalPages: 2 }, + } + ) + ) + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=title'] }) + + await waitFor(async () => expect(await shownTitles()).toEqual(['Alpha', 'beta', 'Gamma 2', 'gamma 10'])) + expect(studiesApi.list).toHaveBeenCalledWith(undefined, expect.objectContaining({ page: 2 })) + expect(screen.getByText('4 studies total')).toBeInTheDocument() + }) + }) }) diff --git a/packages/web/src/pages/__tests__/StudyDetail.test.tsx b/packages/web/src/pages/__tests__/StudyDetail.test.tsx index b573ab39..e2a167ea 100644 --- a/packages/web/src/pages/__tests__/StudyDetail.test.tsx +++ b/packages/web/src/pages/__tests__/StudyDetail.test.tsx @@ -1,14 +1,17 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import { DateFilterProvider } from '../../contexts/DateFilterContext' import StudyDetail from '../StudyDetail' +const routerState = vi.hoisted(() => ({ search: '' })) + vi.mock('react-router-dom', async () => { const actual = await vi.importActual('react-router-dom') return { ...actual, useParams: () => ({ id: '1' }), - useSearchParams: () => [new URLSearchParams(), vi.fn()], + useSearchParams: () => [new URLSearchParams(routerState.search), vi.fn()], } }) @@ -47,6 +50,7 @@ import { studiesApi } from '../../lib/api/studies' describe('StudyDetail page', () => { beforeEach(() => { + routerState.search = '' vi.mocked(api.get).mockResolvedValue({ pagination: { total: 0 } }) vi.mocked(studiesApi.get).mockResolvedValue({ study: { @@ -112,4 +116,40 @@ describe('StudyDetail page', () => { expect(studiesApi.get).toHaveBeenCalled() } ) + + it('sorts Subjects by name naturally, lowest names first', async () => { + // Names drawn from study 70: numeric names and alphanumeric ones. + const names = ['140', '80', '116', '29PH', '10', '1', '100', '350', '2', '3CDX'] + vi.mocked(studiesApi.getSubjects).mockResolvedValue({ + subjects: names.map((name, i) => ({ + id: i + 1, + studyId: 1, + name, + specimenCount: 1, + created: '2024-01-01T00:00:00Z', + lastUpdated: '2024-01-01T00:00:00Z', + })), + }) + routerState.search = 'tab=subjects' + const user = userEvent.setup() + await render( + + + + ) + + const nameHeader = await screen.findByRole('columnheader', { name: 'Name' }) + await screen.findByRole('cell', { name: '3CDX' }) + await user.click(nameHeader) + + const shownNames = () => + screen + .getAllByRole('row') + .slice(1) + .map((row) => within(row).getAllByRole('cell')[0].textContent) + expect(shownNames()).toEqual(['1', '2', '3CDX', '10', '29PH', '80', '100', '116', '140', '350']) + + await user.click(nameHeader) + expect(shownNames()[0]).toBe('350') + }) }) From a7c1ad15fa4fe43326b02805278ad0bd577a59e5 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:33:05 -0700 Subject: [PATCH 03/54] fix(qpcr): record result uploads, plate uploads and deletes atomically (#123) Result upload now writes the run, well results, amplification data and the experiment status in one withWriteTransaction, inserting rows in chunks sized to stay under bun:sqlite's bound-parameter limit. Plate upload and experiment delete run in one transaction each. The qPCR templates are serialized with serializeCsv, so a delimiter in a field is quoted and a missing value is empty. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/qpcr-results-write.ts | 107 +++++++++ packages/api/src/lib/sql-helpers.ts | 11 + .../routes/__tests__/qpcr-experiments.test.ts | 222 +++++++++++++++++- packages/api/src/routes/qpcr-experiments.ts | 167 +++++-------- 4 files changed, 391 insertions(+), 116 deletions(-) create mode 100644 packages/api/src/lib/qpcr-results-write.ts diff --git a/packages/api/src/lib/qpcr-results-write.ts b/packages/api/src/lib/qpcr-results-write.ts new file mode 100644 index 00000000..5d9c4536 --- /dev/null +++ b/packages/api/src/lib/qpcr-results-write.ts @@ -0,0 +1,107 @@ +import { eq } from 'drizzle-orm' +import type { Database } from '../db/client' +import { qpcrAmplificationData, qpcrExperiment, qpcrRun, qpcrWellResult } from '../db/schema' +import { withWriteTransaction } from '../db/write-transaction' +import type { QpcrParseResult } from './qpcr-result-parse' +import { chunkArray, rowsPerInsert } from './sql-helpers' +import { utcNow } from './datetime' + +/** Values each row binds: well results bind 9 (run, well, target, barcode, task, cq, quantities, status), amplification rows 4. */ +const WELL_RESULT_PARAMETERS = 9 +const AMPLIFICATION_PARAMETERS = 4 + +/** Well results and amplification rows join on well position and target, with '' for a missing target. */ +function wellTargetKey(wellPosition: string, targetName: string | null): string { + return `${wellPosition}\t${targetName ?? ''}` +} + +/** + * Record a parsed qPCR result file against an experiment: the run, its well results, their + * amplification data, and the experiment's `results_uploaded` status. All of it commits in one + * transaction, so a failure partway leaves no run or well result behind and a retry starts clean. + */ +export async function recordQpcrResults( + database: Database, + input: { + experimentId: number + instrumentType: string + parseResult: QpcrParseResult + userId: number | null + }, +): Promise { + const { experimentId, instrumentType, parseResult, userId } = input + const now = utcNow() + + // A later row for the same well and target replaces an earlier one, as the unique index requires. + const deduped = new Map() + for (const row of parseResult.wellResults) { + deduped.set(wellTargetKey(row.wellPosition, row.targetName), row) + } + + return withWriteTransaction(database, async (tx) => { + const [runRow] = await tx + .insert(qpcrRun) + .values({ + qpcrExperimentId: experimentId, + instrumentType, + runStartedAt: parseResult.runMetadata.runStartedAt, + runEndedAt: parseResult.runMetadata.runEndedAt, + experimentName: parseResult.runMetadata.experimentName, + fileName: parseResult.runMetadata.fileName, + created: now, + slope: parseResult.runMetadata.slope ?? null, + yIntercept: parseResult.runMetadata.yIntercept ?? null, + rSquared: parseResult.runMetadata.rSquared ?? null, + efficiency: parseResult.runMetadata.efficiency ?? null, + }) + .returning() + + const wellResultIds = new Map() + const wellResultRows = [...deduped.values()].map((row) => ({ + qpcrRunId: runRow.id, + wellPosition: row.wellPosition, + targetName: row.targetName, + sampleBarcode: row.sampleBarcode, + task: row.task, + cq: row.cq, + quantity: row.quantity, + standardQuantity: row.standardQuantity, + ampStatus: row.ampStatus, + })) + for (const chunk of chunkArray(wellResultRows, rowsPerInsert(WELL_RESULT_PARAMETERS))) { + const inserted = await tx + .insert(qpcrWellResult) + .values(chunk) + .returning({ + id: qpcrWellResult.id, + wellPosition: qpcrWellResult.wellPosition, + targetName: qpcrWellResult.targetName, + }) + for (const row of inserted) { + wellResultIds.set(wellTargetKey(row.wellPosition, row.targetName), row.id) + } + } + + const ampRows: Array = [] + for (const row of parseResult.amplificationData) { + const wellResultId = wellResultIds.get(wellTargetKey(row.wellPosition, row.targetName)) + if (wellResultId == null) continue + ampRows.push({ + qpcrWellResultId: wellResultId, + cycle: row.cycle, + rn: row.rn ?? null, + deltaRn: row.deltaRn ?? null, + }) + } + for (const chunk of chunkArray(ampRows, rowsPerInsert(AMPLIFICATION_PARAMETERS))) { + await tx.insert(qpcrAmplificationData).values(chunk) + } + + await tx + .update(qpcrExperiment) + .set({ status: 'results_uploaded', lastUpdated: now, updatedBy: userId }) + .where(eq(qpcrExperiment.id, experimentId)) + + return runRow + }) +} diff --git a/packages/api/src/lib/sql-helpers.ts b/packages/api/src/lib/sql-helpers.ts index 3b8e10fd..e3a01608 100644 --- a/packages/api/src/lib/sql-helpers.ts +++ b/packages/api/src/lib/sql-helpers.ts @@ -18,6 +18,17 @@ export function dateToUpperBound(column: SQLiteColumn, dateTo: string) { */ export const SQL_IN_CHUNK_SIZE = 500 +/** + * Bound parameters one multi-row INSERT may use. bun:sqlite rejects a statement with more + * than 65,535; 8,000 stays well under that and under SQLite's own 32,766 default. + */ +const INSERT_PARAMETER_BUDGET = 8000 + +/** Rows per multi-row INSERT for a table that binds `parametersPerRow` values per row. */ +export function rowsPerInsert(parametersPerRow: number): number { + return Math.max(1, Math.floor(INSERT_PARAMETER_BUDGET / parametersPerRow)) +} + /** Batch an array into chunks for SQLite variable limits. */ export function chunkArray(array: T[], chunkSize: number): T[][] { const chunks: T[][] = [] diff --git a/packages/api/src/routes/__tests__/qpcr-experiments.test.ts b/packages/api/src/routes/__tests__/qpcr-experiments.test.ts index 3a189e01..63db3147 100644 --- a/packages/api/src/routes/__tests__/qpcr-experiments.test.ts +++ b/packages/api/src/routes/__tests__/qpcr-experiments.test.ts @@ -2,8 +2,48 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { setupAuthenticatedRouteTest, type AuthenticatedRouteTestContext } from '../../__tests__/helpers/authenticated-route-test' import { createQpcrExperimentsRoutes } from '../qpcr-experiments' import { utcNow } from '../../lib/datetime' -import { qpcrExperiment, qpcrExperimentTarget, qpcrExperimentWell } from '../../db/schema' -import { qpcrAmplificationData } from '../../db/schema' +import { eq } from 'drizzle-orm' +import { + qpcrAmplificationData, + qpcrExperiment, + qpcrExperimentTarget, + qpcrExperimentWell, + qpcrRun, + qpcrWellResult, +} from '../../db/schema' + +const PLATE_ROWS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] + +/** + * A QuantStudio result workbook laid out like the instrument's export: a Results sheet with one + * row per well and target, and an Amplification Data sheet with one row per well, target and cycle. + */ +async function quantStudioWorkbook(options: { targets: number; cycles: number }): Promise { + const XLSX = await import('xlsx') + const results: Array> = [ + ['Well', 'Well Position', 'Sample Name', 'Target Name', 'Task', 'CT'], + ] + const amplification: Array> = [ + ['Well', 'Well Position', 'Cycle', 'Target Name', 'Rn', 'Delta Rn'], + ] + let well = 0 + for (const row of PLATE_ROWS) { + for (let col = 1; col <= 12; col++) { + well++ + const position = `${row}${col}` + for (let t = 1; t <= options.targets; t++) { + results.push([well, position, `S${well}`, `Target ${t}`, 'UNKNOWN', 25]) + for (let cycle = 1; cycle <= options.cycles; cycle++) { + amplification.push([well, position, cycle, `Target ${t}`, cycle / 10, cycle / 100]) + } + } + } + } + const wb = XLSX.utils.book_new() + XLSX.utils.book_append_sheet(wb, XLSX.utils.aoa_to_sheet(results), 'Results') + XLSX.utils.book_append_sheet(wb, XLSX.utils.aoa_to_sheet(amplification), 'Amplification Data') + return (XLSX.write(wb, { type: 'buffer', bookType: 'xls' }) as Buffer).toString('base64') +} describe('qPCR Experiments Template', () => { let ctx: AuthenticatedRouteTestContext @@ -104,6 +144,145 @@ describe('qPCR Experiments Template', () => { const amp = await ctx.db.select().from(qpcrAmplificationData).all() expect(amp).toHaveLength(2) }) + + async function createExperiment(): Promise { + const res = await ctx.request('/api/qpcr-experiments', { + method: 'POST', + json: { name: 'Large run', templateFormat: 'quant_studio' }, + }) + return ((await res.json()) as { id: number }).id + } + + async function countRows() { + return { + runs: (await ctx.db.select().from(qpcrRun).all()).length, + wellResults: (await ctx.db.select().from(qpcrWellResult).all()).length, + amplification: (await ctx.db.select().from(qpcrAmplificationData).all()).length, + } + } + + it('records an upload with more than 20,000 amplification rows', async () => { + // 96 wells x 5 targets x 42 cycles = 20,160 rows, 80,640 bound values unchunked. + const fileContent = await quantStudioWorkbook({ targets: 5, cycles: 42 }) + const id = await createExperiment() + + const res = await ctx.request(`/api/qpcr-experiments/${id}/results`, { + method: 'POST', + json: { fileContent, fileName: 'large.xls', instrumentType: 'QuantStudio' }, + }) + + expect(res.status).toBe(201) + const body = (await res.json()) as { amplificationCount: number; wellResultCount: number } + expect(body.amplificationCount).toBe(20_160) + expect(body.wellResultCount).toBe(480) + expect(await countRows()).toEqual({ runs: 1, wellResults: 480, amplification: 20_160 }) + const exp = await ctx.db.select().from(qpcrExperiment).where(eq(qpcrExperiment.id, id)).get() + expect(exp?.status).toBe('results_uploaded') + }) + + it('leaves no run or well results behind when an insert fails partway, and a retry records one run', async () => { + const fileContent = await quantStudioWorkbook({ targets: 5, cycles: 42 }) + const id = await createExperiment() + // Fail on the plate's last amplification row, after the run, every well result and the earlier chunks are written. + ctx.sqlite.exec(` + CREATE TRIGGER fail_last_amplification BEFORE INSERT ON qpcr_amplification_data + WHEN NEW.cycle = 42 AND (SELECT well_position || target_name FROM qpcr_well_result WHERE id = NEW.qpcr_well_result_id) = 'H12Target 5' + BEGIN SELECT RAISE(ABORT, 'simulated failure'); END + `) + + const failed = await ctx.request(`/api/qpcr-experiments/${id}/results`, { + method: 'POST', + json: { fileContent, fileName: 'large.xls', instrumentType: 'QuantStudio' }, + }) + + expect(failed.status).toBe(500) + expect(await countRows()).toEqual({ runs: 0, wellResults: 0, amplification: 0 }) + const unchanged = await ctx.db.select().from(qpcrExperiment).where(eq(qpcrExperiment.id, id)).get() + expect(unchanged?.status).toBe('setup') + + ctx.sqlite.exec('DROP TRIGGER fail_last_amplification') + const retried = await ctx.request(`/api/qpcr-experiments/${id}/results`, { + method: 'POST', + json: { fileContent, fileName: 'large.xls', instrumentType: 'QuantStudio' }, + }) + + expect(retried.status).toBe(201) + expect(await countRows()).toEqual({ runs: 1, wellResults: 480, amplification: 20_160 }) + }) + }) + + describe('DELETE /:id', () => { + async function experimentWithResults(): Promise { + const created = (await ( + await ctx.request('/api/qpcr-experiments', { method: 'POST', json: { name: 'Delete', templateFormat: 'quant_studio' } }) + ).json()) as { id: number } + await ctx.db.insert(qpcrExperimentWell).values({ qpcrExperimentId: created.id, wellPosition: 'A01', contentType: 'unknown' }) + const res = await ctx.request(`/api/qpcr-experiments/${created.id}/results`, { + method: 'POST', + json: { fileContent: await quantStudioWorkbook({ targets: 1, cycles: 2 }), fileName: 'run.xls', instrumentType: 'QuantStudio' }, + }) + expect(res.status).toBe(201) + return created.id + } + + it('deletes the experiment with its wells, runs, well results and amplification data', async () => { + const id = await experimentWithResults() + + const res = await ctx.request(`/api/qpcr-experiments/${id}`, { method: 'DELETE' }) + + expect(res.status).toBe(204) + expect(await ctx.db.select().from(qpcrExperiment).all()).toHaveLength(0) + expect(await ctx.db.select().from(qpcrExperimentWell).all()).toHaveLength(0) + expect(await ctx.db.select().from(qpcrRun).all()).toHaveLength(0) + expect(await ctx.db.select().from(qpcrWellResult).all()).toHaveLength(0) + expect(await ctx.db.select().from(qpcrAmplificationData).all()).toHaveLength(0) + }) + + it('keeps every related row when the delete fails partway', async () => { + const id = await experimentWithResults() + ctx.sqlite.exec(` + CREATE TRIGGER fail_experiment_delete BEFORE DELETE ON qpcr_experiment + BEGIN SELECT RAISE(ABORT, 'simulated failure'); END + `) + + const res = await ctx.request(`/api/qpcr-experiments/${id}`, { method: 'DELETE' }) + + expect(res.status).toBe(500) + expect(await ctx.db.select().from(qpcrExperiment).all()).toHaveLength(1) + expect(await ctx.db.select().from(qpcrExperimentWell).all()).toHaveLength(1) + expect(await ctx.db.select().from(qpcrRun).all()).toHaveLength(1) + expect(await ctx.db.select().from(qpcrWellResult).all()).toHaveLength(96) + expect(await ctx.db.select().from(qpcrAmplificationData).all()).toHaveLength(192) + }) + }) + + describe('POST /:id/plate', () => { + it('keeps the previous wells and status when the upload fails partway', async () => { + const created = (await ( + await ctx.request('/api/qpcr-experiments', { method: 'POST', json: { name: 'Plate', templateFormat: 'biorad' } }) + ).json()) as { id: number } + await ctx.db.insert(qpcrExperimentWell).values({ + qpcrExperimentId: created.id, + wellPosition: 'A01', + barcode: 'OLD-1', + contentType: 'unknown', + }) + ctx.sqlite.exec(` + CREATE TRIGGER fail_status_change BEFORE UPDATE OF status ON qpcr_experiment + BEGIN SELECT RAISE(ABORT, 'simulated failure'); END + `) + + const res = await ctx.request(`/api/qpcr-experiments/${created.id}/plate`, { + method: 'POST', + json: { csvText: 'Tube ID,Position\nNEW-1,A01\n', scannerConfigurationId: 'traxcer' }, + }) + + expect(res.status).toBe(500) + const wells = await ctx.db.select().from(qpcrExperimentWell).all() + expect(wells.map((w) => w.barcode)).toEqual(['OLD-1']) + const exp = await ctx.db.select().from(qpcrExperiment).where(eq(qpcrExperiment.id, created.id)).get() + expect(exp?.status).toBe('setup') + }) }) describe('GET /:id/template', () => { @@ -214,6 +393,45 @@ describe('qPCR Experiments Template', () => { expect(quantCols[2]).toBe('MT-001') }) + it('quotes fields that hold a delimiter or quote and writes missing values as empty', async () => { + const [exp] = await ctx.db + .insert(qpcrExperiment) + .values({ + name: 'Quoting', + templateFormat: 'biorad', + status: 'in_progress', + created: utcNow(), + lastUpdated: utcNow(), + }) + .returning() + if (!exp) throw new Error('Insert failed') + await ctx.db.insert(qpcrExperimentTarget).values({ + qpcrExperimentId: exp.id, + targetName: 'var,ATS "v2"\tB', + fluorophore: null, + reporter: null, + sortOrder: 0, + }) + await ctx.db.insert(qpcrExperimentWell).values({ + qpcrExperimentId: exp.id, + wellPosition: 'A01', + barcode: 'MT,001', + contentType: 'unknown', + }) + + const biorad = await (await ctx.request(`/api/qpcr-experiments/${exp.id}/template?format=biorad`)).text() + expect(biorad).toBe( + 'Well,Fluorophore,Target Name,Content,Sample Name,Quantity\n' + + 'A1,,"var,ATS ""v2""\tB",Unk,"MT,001",', + ) + + const quant = await (await ctx.request(`/api/qpcr-experiments/${exp.id}/template?format=quant_studio`)).text() + const lines = quant.split('\n') + expect(lines).toContain('* Instrument Type = ') + expect(lines.at(-1)).toBe('1\tA1\tMT,001\t"var,ATS ""v2""\tB"\tUNKNOWN\t\tNFQ-MGB\t') + expect(quant).not.toContain('null') + }) + it('returns 400 when experiment has no targets', async () => { const [exp] = await ctx.db .insert(qpcrExperiment) diff --git a/packages/api/src/routes/qpcr-experiments.ts b/packages/api/src/routes/qpcr-experiments.ts index d1e1e526..1f9ff14f 100644 --- a/packages/api/src/routes/qpcr-experiments.ts +++ b/packages/api/src/routes/qpcr-experiments.ts @@ -28,6 +28,8 @@ import { normalizeWellPosition, parsePlateCSV, validateWellPosition } from '../l import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' import { withWriteTransaction } from '../db/write-transaction' +import { recordQpcrResults } from '../lib/qpcr-results-write' +import { serializeCsv, type CsvCellValue, type CSVExportOptions } from '@sampledb/contract' /** * Attach the Source (subject vs control provenance) to each well, resolving @@ -46,6 +48,12 @@ async function attachWellSources( } const ROWS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] + +/** + * Instrument software imports the template, so it keeps its plain LF layout with no BOM and no + * formula neutralizing; fields are still quoted, and a missing value is written empty. + */ +const TEMPLATE_CSV_OPTIONS: CSVExportOptions = { bom: false, lineEnding: 'lf' } const COLS = 12 export function createQpcrExperimentsRoutes(database: Database): Hono { @@ -233,30 +241,24 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { const exp = await database.select().from(qpcrExperiment).where(eq(qpcrExperiment.id, id)).get() if (!exp) return c.json({ error: 'Not found' }, 404) - const runs = await database - .select({ id: qpcrRun.id }) - .from(qpcrRun) - .where(eq(qpcrRun.qpcrExperimentId, id)) - const runIds = runs.map((r) => r.id) - if (runIds.length > 0) { - const wellResults = await database + // Subqueries rather than id lists, so an experiment with many runs stays under the parameter limit. + const deleted = await withWriteTransaction(database, async (tx) => { + const runIds = tx + .select({ id: qpcrRun.id }) + .from(qpcrRun) + .where(eq(qpcrRun.qpcrExperimentId, id)) + const wellResultIds = tx .select({ id: qpcrWellResult.id }) .from(qpcrWellResult) .where(inArray(qpcrWellResult.qpcrRunId, runIds)) - const wellResultIds = wellResults.map((w) => w.id) - if (wellResultIds.length > 0) { - await database - .delete(qpcrAmplificationData) - .where(inArray(qpcrAmplificationData.qpcrWellResultId, wellResultIds)) - } - await database.delete(qpcrWellResult).where(inArray(qpcrWellResult.qpcrRunId, runIds)) - } - await database.delete(qpcrRun).where(eq(qpcrRun.qpcrExperimentId, id)) - await database.delete(qpcrExperimentWell).where(eq(qpcrExperimentWell.qpcrExperimentId, id)) - const deleted = await database - .delete(qpcrExperiment) - .where(eq(qpcrExperiment.id, id)) - .returning() + await tx + .delete(qpcrAmplificationData) + .where(inArray(qpcrAmplificationData.qpcrWellResultId, wellResultIds)) + await tx.delete(qpcrWellResult).where(inArray(qpcrWellResult.qpcrRunId, runIds)) + await tx.delete(qpcrRun).where(eq(qpcrRun.qpcrExperimentId, id)) + await tx.delete(qpcrExperimentWell).where(eq(qpcrExperimentWell.qpcrExperimentId, id)) + return tx.delete(qpcrExperiment).where(eq(qpcrExperiment.id, id)).returning() + }) if (deleted.length === 0) { return c.json({ error: 'Not found' }, 404) } @@ -477,20 +479,6 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { const now = utcNow() const userId = (c.get('user') as { id: number } | undefined)?.id ?? null - await database.delete(qpcrExperimentWell).where(eq(qpcrExperimentWell.qpcrExperimentId, id)) - if (wellsToInsert.length > 0) { - await database.insert(qpcrExperimentWell).values( - wellsToInsert.map((w) => ({ - qpcrExperimentId: w.qpcrExperimentId, - wellPosition: w.wellPosition, - barcode: w.barcode, - storageContainerId: w.storageContainerId, - specimenId: w.specimenId, - contentType: w.contentType, - standardDensity: w.standardDensity, - })) - ) - } // After plate upload, set status to in_progress and optionally update plate barcode const experimentUpdates: { status: string; plateBarcode?: string; lastUpdated: string; updatedBy: number | null } = { status: 'in_progress', @@ -500,7 +488,14 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { if (data.plateBarcode !== undefined && data.plateBarcode !== null) { experimentUpdates.plateBarcode = data.plateBarcode } - await database.update(qpcrExperiment).set(experimentUpdates).where(eq(qpcrExperiment.id, id)) + // Replace the wells and advance the status together, so a failure keeps the previous layout. + await withWriteTransaction(database, async (tx) => { + await tx.delete(qpcrExperimentWell).where(eq(qpcrExperimentWell.qpcrExperimentId, id)) + if (wellsToInsert.length > 0) { + await tx.insert(qpcrExperimentWell).values(wellsToInsert) + } + await tx.update(qpcrExperiment).set(experimentUpdates).where(eq(qpcrExperiment.id, id)) + }) const inserted = await database .select() @@ -648,74 +643,12 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { parseResult = await parseQuantStudioXls(buffer, data.fileName) } - const now = utcNow() - const [runRow] = await database - .insert(qpcrRun) - .values({ - qpcrExperimentId: id, - instrumentType: data.instrumentType, - runStartedAt: parseResult.runMetadata.runStartedAt, - runEndedAt: parseResult.runMetadata.runEndedAt, - experimentName: parseResult.runMetadata.experimentName, - fileName: parseResult.runMetadata.fileName, - created: now, - slope: parseResult.runMetadata.slope ?? null, - yIntercept: parseResult.runMetadata.yIntercept ?? null, - rSquared: parseResult.runMetadata.rSquared ?? null, - efficiency: parseResult.runMetadata.efficiency ?? null, - }) - .returning() - const runId = runRow.id - - const wellResultKeys = new Map() - const deduped = new Map() - for (const row of parseResult.wellResults) { - const key = `${row.wellPosition}\t${row.targetName ?? ''}` - deduped.set(key, row) - } - for (const row of deduped.values()) { - // Must match the amplification lookup below, which uses '' for a missing target. - const targetKey = row.targetName ?? '' - const [inserted] = await database - .insert(qpcrWellResult) - .values({ - qpcrRunId: runId, - wellPosition: row.wellPosition, - targetName: row.targetName, - sampleBarcode: row.sampleBarcode, - task: row.task, - cq: row.cq, - quantity: row.quantity, - standardQuantity: row.standardQuantity, - ampStatus: row.ampStatus, - }) - .returning() - wellResultKeys.set(`${row.wellPosition}\t${targetKey}`, inserted.id) - } - - if (parseResult.amplificationData.length > 0) { - const ampRows: Array<{ qpcrWellResultId: number; cycle: number; rn: number | null; deltaRn: number | null }> = [] - for (const row of parseResult.amplificationData) { - const key = `${row.wellPosition}\t${row.targetName ?? ''}` - const wellResultId = wellResultKeys.get(key) - if (wellResultId != null) { - ampRows.push({ - qpcrWellResultId: wellResultId, - cycle: row.cycle, - rn: row.rn ?? null, - deltaRn: row.deltaRn ?? null, - }) - } - } - if (ampRows.length > 0) { - await database.insert(qpcrAmplificationData).values(ampRows) - } - } - - await database - .update(qpcrExperiment) - .set({ status: 'results_uploaded', lastUpdated: now, updatedBy: (c.get('user') as { id: number } | undefined)?.id ?? null }) - .where(eq(qpcrExperiment.id, id)) + const runRow = await recordQpcrResults(database, { + experimentId: id, + instrumentType: data.instrumentType, + parseResult, + userId: (c.get('user') as { id: number } | undefined)?.id ?? null, + }) return c.json({ run: runRow, wellResultCount: parseResult.wellResults.length, amplificationCount: parseResult.amplificationData.length }, 201) } catch (error) { @@ -840,8 +773,7 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { const instrumentType = exp.instrumentType if (format === 'biorad') { - const header = 'Well,Fluorophore,Target Name,Content,Sample Name,Quantity' - const lines: string[] = [header] + const rows: CsvCellValue[][] = [] for (const row of ROWS) { for (let col = 1; col <= COLS; col++) { const pos = `${row}${col.toString().padStart(2, '0')}` @@ -854,12 +786,15 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { const quantity = well.contentType === 'standard' && well.standardDensity != null ? String(well.standardDensity) : '' const wellA1 = wellPositionToA1(pos) for (const t of targets) { - const fluorophore = t.fluorophore - lines.push(`${wellA1},${fluorophore},${t.targetName},${content},${sampleName},${quantity}`) + rows.push([wellA1, t.fluorophore, t.targetName, content, sampleName, quantity]) } } } - const csv = lines.join('\n') + const csv = serializeCsv( + ['Well', 'Fluorophore', 'Target Name', 'Content', 'Sample Name', 'Quantity'], + rows, + TEMPLATE_CSV_OPTIONS, + ) return new Response(csv, { status: 200, headers: { @@ -872,12 +807,11 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { const meta = [ '* Block Type = 96-Well', '* Experiment Type = Standard Curve', - `* Instrument Type = ${instrumentType}`, + `* Instrument Type = ${instrumentType ?? ''}`, '* No. Of Wells = 96', '* Set Up Well Section Info =', ] - const dataHeader = 'Well\tWell Position\tSample Name\tTarget Name\tTask\tReporter\tQuencher\tQuantity' - const dataLines: string[] = [dataHeader] + const dataRows: CsvCellValue[][] = [] for (const row of ROWS) { for (let col = 1; col <= COLS; col++) { const pos = `${row}${col.toString().padStart(2, '0')}` @@ -897,11 +831,16 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { if (r.toLowerCase() === 'sybr') return 'None' return 'NFQ-MGB' })() - dataLines.push(`${wellIdx}\t${wellPos}\t${sampleName}\t${t.targetName}\t${task}\t${reporter}\t${quencher}\t${quantity}`) + dataRows.push([wellIdx, wellPos, sampleName, t.targetName, task, reporter, quencher, quantity]) } } } - const tsv = [...meta, '', dataLines.join('\n')].join('\n') + const dataSection = serializeCsv( + ['Well', 'Well Position', 'Sample Name', 'Target Name', 'Task', 'Reporter', 'Quencher', 'Quantity'], + dataRows, + { ...TEMPLATE_CSV_OPTIONS, delimiter: '\t' }, + ) + const tsv = [...meta, '', dataSection].join('\n') return new Response(tsv, { status: 200, headers: { From b4144a208fdb139a3a9e0557415f4a86a994e228 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:33:05 -0700 Subject: [PATCH 04/54] fix(web): read bulk import Collection names and barcodes from their columns (#114) The import wizard decided a missing Collection identifier was a barcode by its shape (uppercase, hyphens, longer than five characters), so a plate_name of PLATE-001 became an empty plate "Collection-PLATE-001" while the server created a second PLATE-001 holding the Specimens. Each row now yields one Collection reference whose name comes from plate_name/box_name/bag_name and whose barcode comes from collection_barcode. A row giving both makes one Collection, and a Collection given only a barcode is named after it, as the server's planner does. The Papers template example row now puts BOX-003 in box_name, leaves bag_name empty, and fills sheet_name and sublabel. The required-fields list shows "box_name or bag_name". Every template is now tested against the required fields, the contract's CSV row rules, and the bulk-combined validate schema. Co-Authored-By: Claude Opus 5.5 --- .../docs/guides/bulk-operations/import.md | 2 +- .../__tests__/helpers/mock-api-templates.ts | 11 +- .../web/src/components/BulkImportFlow.tsx | 33 ++--- .../__tests__/BulkImportFlow.test.tsx | 133 +++++++++++++++++- .../__tests__/useBulkImportWorkflow.test.tsx | 109 ++++++++++++-- .../web/src/hooks/useBulkImportWorkflow.ts | 117 ++++++++------- .../src/lib/__tests__/bulk-import-csv.test.ts | 64 +++++++++ .../__tests__/bulk-import-validation.test.ts | 39 ++++- packages/web/src/lib/bulk-import-csv.ts | 3 +- .../web/src/lib/bulk-import-validation.ts | 42 ++++-- 10 files changed, 435 insertions(+), 118 deletions(-) diff --git a/packages/docs/src/content/docs/guides/bulk-operations/import.md b/packages/docs/src/content/docs/guides/bulk-operations/import.md index 74b58a6f..9392ac29 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/import.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/import.md @@ -71,7 +71,7 @@ Column headers are case-insensitive. `Position`, `position`, `well_position`, an **Specimens Only or Combined, no containers:** `study_short_code`, `subject_name`, `specimen_type_name`. Optional `collection_date` as YYYY-MM-DD. US-style M/D/YYYY and YYYY/MM/DD are also accepted and stored as YYYY-MM-DD; impossible dates such as 2024-02-30 are rejected. -**With containers:** the same columns, plus container fields for the selected type. Optional `comment` is stored on each container. Collection identifier must use the column for that type (`plate_name`, `box_name`, or `bag_name`) so it matches the templates. +**With containers:** the same columns, plus container fields for the selected type. Optional `comment` is stored on each container. Collection identifier must use the column for that type (`plate_name`, `box_name`, or `bag_name`) so it matches the templates. A value in that column is always a name, whatever it looks like: `PLATE-001` creates a plate named `PLATE-001`. For plates and cryovial boxes, an optional `collection_barcode` column gives the collection's barcode; a new collection given only a barcode is named after it. - Micronix: `plate_name`, unique barcode, `position` (letter A-H plus two digits, for example `A01`). - Cryovial: `box_name`, `position`. Barcode is optional. diff --git a/packages/web/src/__tests__/helpers/mock-api-templates.ts b/packages/web/src/__tests__/helpers/mock-api-templates.ts index dc1dd187..a8d7c548 100644 --- a/packages/web/src/__tests__/helpers/mock-api-templates.ts +++ b/packages/web/src/__tests__/helpers/mock-api-templates.ts @@ -506,11 +506,20 @@ export function bulkImportFlowMock(): MockApiOverrides { bulkCombinedValidate: vi.fn().mockResolvedValue({ valid: true, errors: [] }), }, collectionsApi: { - check: vi.fn().mockResolvedValue({ results: [] }), + // One result per Collection checked, as the API returns; every Collection exists. + check: vi.fn((data: { collections: Array<{ identifier: string; type: string }> }) => + Promise.resolve({ + results: data.collections.map((c) => ({ ...c, exists: true, id: 1 })), + }), + ), listMicronixPlates: vi.fn().mockResolvedValue([]), listCryovialBoxes: vi.fn().mockResolvedValue([]), listBoxes: vi.fn().mockResolvedValue([]), listBags: vi.fn().mockResolvedValue([]), + createMicronixPlate: vi.fn(), + createCryovialBox: vi.fn(), + createBox: vi.fn(), + createBag: vi.fn(), }, } } diff --git a/packages/web/src/components/BulkImportFlow.tsx b/packages/web/src/components/BulkImportFlow.tsx index a247dd62..d171f27d 100644 --- a/packages/web/src/components/BulkImportFlow.tsx +++ b/packages/web/src/components/BulkImportFlow.tsx @@ -16,8 +16,7 @@ import { getBulkImportOptionalFields, parseBulkImportCSV, mapBulkImportRowsToPayload, - getBulkImportRowCollectionName, - getBulkImportRowCollectionCheck, + getBulkImportRowCollectionRef, type CSVRow, type BulkImportValidationError, type ImportType as LibImportType, @@ -171,9 +170,6 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm } } - const getRowCollectionName = (row: CSVRow) => - getBulkImportRowCollectionName(row, containerType) - const downloadTemplate = () => { if (templateSpecimenTypes.isError) { return @@ -271,19 +267,10 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm const collectionType = getCollectionType() if (collectionType || containerType === 'paper') { try { - const missing = await fetchBulkImportMissingCollections( - containerType === 'paper' - ? { - rows, - getRowCollectionCheck: (row) => - getBulkImportRowCollectionCheck(row, containerType), - } - : { - rows, - collectionType: collectionType!, - getRowCollectionName, - } - ) + const missing = await fetchBulkImportMissingCollections({ + rows, + getRowCollectionRef: (row) => getBulkImportRowCollectionRef(row, containerType), + }) setMissingCollections(missing) if (missing.length > 0) { @@ -339,8 +326,10 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setMissingCollections([...updated]) try { - const name = collection.name || (collection.barcode ? `Collection-${collection.barcode}` : `Collection-${Date.now()}`) - const barcode = collection.barcode || collection.collectionBarcode + const barcode = collection.barcode || collection.collectionBarcode || undefined + // A Collection the CSV gives only by barcode is named after it, as the import itself + // does. Every missing Collection has a name or a barcode. + const name = collection.name || collection.barcode || '' const itemCollectionType = collection.collectionType ?? defaultCollectionType if (itemCollectionType === 'micronix_plate') { @@ -667,7 +656,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm {Object.keys(preview[0] ?? {}).map((key) => { const required = getRequiredFields() - const isRequired = required.includes(key) + const isRequired = required.some((field) => field.split(' or ').includes(key)) return ( {key} @@ -754,7 +743,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm

Barcode: {collection.barcode || collection.collectionBarcode}

)} {!collection.name && (collection.barcode || collection.collectionBarcode) && ( -

A name will be generated from the barcode

+

Named after its barcode

)}
{collection.status === 'success' && ✓ Created} diff --git a/packages/web/src/components/__tests__/BulkImportFlow.test.tsx b/packages/web/src/components/__tests__/BulkImportFlow.test.tsx index 3bf3ccd9..bcc3a8cf 100644 --- a/packages/web/src/components/__tests__/BulkImportFlow.test.tsx +++ b/packages/web/src/components/__tests__/BulkImportFlow.test.tsx @@ -1,9 +1,18 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest' +import { describe, it, expect, vi, beforeEach, beforeAll, afterAll } from 'vitest' import { render, screen, fireEvent, waitFor, act } from '../../__tests__/helpers/render' import userEvent from '@testing-library/user-event' import BulkImportFlow from '../BulkImportFlow' import { importsApi } from '../../lib/api/imports' import { specimenTypesApi } from '../../lib/api/reference-data' +import { collectionsApi } from '../../lib/api/collections' + +vi.mock('../LocationPicker', () => ({ + default: ({ onChange }: { onChange: (id: number | null) => void }) => ( + + ), +})) vi.mock('../../lib/api/reference-data', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -165,4 +174,126 @@ describe('BulkImportFlow', () => { }) }) + describe('creating missing Collections (combined, per subject)', () => { + /** Upload `text` as a combined micronix import with per-subject atomicity and validate it. */ + async function validateCsv(text: string) { + const user = userEvent.setup() + await render(, { initialEntries: ['/import?step=upload'] }) + await user.selectOptions(screen.getByLabelText(/import type/i), 'combined') + await user.selectOptions(screen.getByLabelText(/container type/i), 'micronix_tube') + await user.click(screen.getByRole('radio', { name: /per subject/i })) + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + await user.upload(fileInput, new File([text], 'i.csv', { type: 'text/csv' })) + // jsdom's constraint validation does not see user-event's file on the required input, + // so clicking Validate & Continue would not submit; submit the form directly. + await act(async () => { + fireEvent.submit(fileInput.closest('form')!) + }) + return user + } + + // jsdom's File has no text(); read the uploaded file with FileReader instead. + const fileProto = File.prototype as { text?: () => Promise } + const originalText = fileProto.text + beforeAll(() => { + fileProto.text = function (this: File) { + return new Promise((resolve) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.readAsText(this) + }) + } + }) + afterAll(() => { + if (originalText) fileProto.text = originalText + else delete fileProto.text + }) + + beforeEach(() => { + vi.mocked(importsApi.bulkCombinedValidate).mockResolvedValue({ valid: true, errors: [] }) + vi.mocked(collectionsApi.check).mockImplementation(({ collections }) => + Promise.resolve({ + results: collections.map((c) => ({ ...c, exists: false, id: null })), + }), + ) + vi.mocked(collectionsApi.createMicronixPlate).mockResolvedValue({ + plate: {} as Awaited>['plate'], + }) + }) + + it('creates one plate named after a barcode-shaped plate_name', async () => { + const user = await validateCsv( + 'subject_name,specimen_type_name,plate_name,barcode,position\n' + + 'S1,Serum,PLATE-001,MT1,A01\n' + + 'S2,Serum,PLATE-001,MT2,A02', + ) + + expect(await screen.findByRole('heading', { name: 'PLATE-001' })).toBeInTheDocument() + expect(collectionsApi.check).toHaveBeenCalledWith({ + collections: [{ identifier: 'PLATE-001', type: 'micronix_plate' }], + }) + expect(screen.queryByText(/named after its barcode/i)).not.toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Select location 10' })) + await user.click(screen.getByRole('button', { name: /create collections & continue/i })) + + await waitFor(() => expect(importsApi.bulkCombined).toHaveBeenCalled()) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledTimes(1) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledWith({ + name: 'PLATE-001', + locationId: 10, + barcode: undefined, + }) + const request = vi.mocked(importsApi.bulkCombined).mock.calls[0]![0] + const collections = request.subjects.flatMap((s) => s.specimens.map((sp) => sp.container?.collection)) + expect(collections).toEqual([ + expect.objectContaining({ type: 'micronix_plate', name: 'PLATE-001' }), + expect.objectContaining({ type: 'micronix_plate', name: 'PLATE-001' }), + ]) + for (const collection of collections) expect(collection).not.toHaveProperty('barcode') + }) + + it('creates one plate with the name from plate_name and the barcode from collection_barcode', async () => { + const user = await validateCsv( + 'subject_name,specimen_type_name,plate_name,collection_barcode,barcode,position\n' + + 'S1,Serum,My Plate,PB-778899,MT1,A01\n' + + 'S2,Serum,My Plate,PB-778899,MT2,A02', + ) + + expect(await screen.findByRole('heading', { name: 'My Plate' })).toBeInTheDocument() + expect(screen.getByText('Barcode: PB-778899')).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Select location 10' })) + await user.click(screen.getByRole('button', { name: /create collections & continue/i })) + + await waitFor(() => expect(importsApi.bulkCombined).toHaveBeenCalled()) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledTimes(1) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledWith({ + name: 'My Plate', + locationId: 10, + barcode: 'PB-778899', + }) + }) + + it('names a plate given only by collection_barcode after its barcode', async () => { + const user = await validateCsv( + 'subject_name,specimen_type_name,collection_barcode,barcode,position\n' + + 'S1,Serum,plate 7,MT1,A01', + ) + + expect(await screen.findByText('Barcode: plate 7')).toBeInTheDocument() + expect(screen.getByText(/named after its barcode/i)).toBeInTheDocument() + + await user.click(screen.getByRole('button', { name: 'Select location 10' })) + await user.click(screen.getByRole('button', { name: /create collections & continue/i })) + + await waitFor(() => expect(importsApi.bulkCombined).toHaveBeenCalled()) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledTimes(1) + expect(collectionsApi.createMicronixPlate).toHaveBeenCalledWith({ + name: 'plate 7', + locationId: 10, + barcode: 'plate 7', + }) + }) + }) }) diff --git a/packages/web/src/hooks/__tests__/useBulkImportWorkflow.test.tsx b/packages/web/src/hooks/__tests__/useBulkImportWorkflow.test.tsx index e05e8716..0bf5b1d6 100644 --- a/packages/web/src/hooks/__tests__/useBulkImportWorkflow.test.tsx +++ b/packages/web/src/hooks/__tests__/useBulkImportWorkflow.test.tsx @@ -8,6 +8,8 @@ import { fetchBulkImportMissingCollections, useBulkImportTemplateSpecimenTypes, } from '../useBulkImportWorkflow' +import { getBulkImportRowCollectionRef, type CSVRow } from '../../lib/bulk-import-validation' +import type { ContainerType } from '../../lib/container-types' vi.mock('../../lib/api/reference-data', () => ({ specimenTypesApi: { @@ -74,25 +76,104 @@ describe('useBulkImportWorkflow', () => { }) describe('fetchBulkImportMissingCollections', () => { - it('returns missing collections from check API', async () => { - const missing = await fetchBulkImportMissingCollections({ - rows: [{ plate_name: 'My Plate' }], - collectionType: 'micronix_plate', - getRowCollectionName: (row) => row.plate_name, + /** The check API reports every Collection missing except those identified by `existing`. */ + function checkReports(existing: string[] = []) { + vi.mocked(collectionsApi.check).mockImplementation(({ collections }) => + Promise.resolve({ + results: collections.map((c) => ({ + ...c, + exists: existing.includes(c.identifier), + id: existing.includes(c.identifier) ? 1 : null, + })), + }), + ) + } + + const missingFor = (rows: CSVRow[], containerType: ContainerType) => + fetchBulkImportMissingCollections({ + rows, + getRowCollectionRef: (row) => getBulkImportRowCollectionRef(row, containerType), }) - expect(missing).toHaveLength(1) - expect(missing[0]?.name).toBe('My Plate') + + // Name-shaped and barcode-shaped values; the column decides which a value is. + const values = ['My Plate', 'PLATE-001', 'REVIEWPLATE-01', 'ABC123456'] + + it.each([ + ['plate_name', 'micronix_tube', 'micronix_plate'], + ['plate_name', 'static_well', 'micronix_plate'], + ['box_name', 'cryovial_tube', 'cryovial_box'], + ['box_name', 'paper', 'box'], + ['bag_name', 'paper', 'bag'], + ] as const)('treats every value in %s (%s) as a name', async (column, containerType, type) => { + checkReports() + for (const value of values) { + const missing = await missingFor([{ [column]: value }, { [column]: value }], containerType) + expect(missing).toEqual([{ name: value, collectionType: type, locationId: null, status: 'pending' }]) + } + }) + + it.each([ + ['micronix_tube', 'micronix_plate'], + ['static_well', 'micronix_plate'], + ['cryovial_tube', 'cryovial_box'], + ] as const)('treats every value in collection_barcode (%s) as a barcode', async (containerType, type) => { + checkReports() + for (const value of values) { + const missing = await missingFor([{ collection_barcode: value }], containerType) + expect(missing).toEqual([ + { name: '', collectionType: type, barcode: value, locationId: null, status: 'pending' }, + ]) + } + }) + + it('makes one Collection of a name and barcode given together', async () => { + checkReports() + const missing = await missingFor( + [ + { plate_name: 'PLATE-001', collection_barcode: 'Barcode 7' }, + { plate_name: 'PLATE-001', collection_barcode: 'Barcode 7' }, + ], + 'micronix_tube', + ) + expect(missing).toEqual([ + { name: 'PLATE-001', collectionType: 'micronix_plate', barcode: 'Barcode 7', locationId: null, status: 'pending' }, + ]) + expect(collectionsApi.check).toHaveBeenCalledWith({ + collections: [ + { identifier: 'PLATE-001', type: 'micronix_plate' }, + { identifier: 'Barcode 7', type: 'micronix_plate' }, + ], + }) + }) + + it('leaves out a Collection found by its name or its barcode', async () => { + checkReports(['PLATE-001', 'PB-2']) + const missing = await missingFor( + [ + { plate_name: 'PLATE-001' }, + { plate_name: 'Plate Two', collection_barcode: 'PB-2' }, + { plate_name: 'PLATE-003' }, + ], + 'micronix_tube', + ) + expect(missing.map((m) => m.name)).toEqual(['PLATE-003']) + }) + + it('ignores collection_barcode on paper rows', async () => { + checkReports() + const missing = await missingFor([{ box_name: 'BOX-003', collection_barcode: 'X-1' }], 'paper') + expect(missing).toEqual([{ name: 'BOX-003', collectionType: 'box', locationId: null, status: 'pending' }]) + }) + + it('does not call the check API when no row names a Collection', async () => { + const missing = await missingFor([{ plate_name: '' }], 'micronix_tube') + expect(missing).toEqual([]) + expect(collectionsApi.check).not.toHaveBeenCalled() }) it('throws when check API fails', async () => { vi.mocked(collectionsApi.check).mockRejectedValue(new Error('Network error')) - await expect( - fetchBulkImportMissingCollections({ - rows: [{ plate_name: 'PLATE1' }], - collectionType: 'micronix_plate', - getRowCollectionName: (row) => row.plate_name, - }), - ).rejects.toThrow() + await expect(missingFor([{ plate_name: 'PLATE1' }], 'micronix_tube')).rejects.toThrow() }) }) }) diff --git a/packages/web/src/hooks/useBulkImportWorkflow.ts b/packages/web/src/hooks/useBulkImportWorkflow.ts index fedabbe4..d8a3804b 100644 --- a/packages/web/src/hooks/useBulkImportWorkflow.ts +++ b/packages/web/src/hooks/useBulkImportWorkflow.ts @@ -3,7 +3,12 @@ import { collectionsApi } from '../lib/api/collections' import { specimenTypesApi } from '../lib/api/reference-data' import { getQueryErrorMessage } from '../ui' import type { ContainerType } from '../components/ContainerRegistration' -import type { CSVRow, ImportType } from '../lib/bulk-import-validation' +import type { + BulkImportCollectionRef, + BulkImportCollectionType, + CSVRow, + ImportType, +} from '../lib/bulk-import-validation' export const bulkImportKeys = { all: ['bulk-import'] as const, @@ -13,9 +18,12 @@ export const bulkImportKeys = { } export type BulkImportMissingCollection = { + /** From the CSV name column; empty when the rows give only a Collection barcode. */ name: string - collectionType?: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' + collectionType?: BulkImportCollectionType + /** From the CSV collection_barcode column. */ barcode?: string + /** Typed in the wizard when the CSV gives no Collection barcode. */ collectionBarcode?: string locationId: number | null status: 'pending' | 'creating' | 'success' | 'error' @@ -65,78 +73,63 @@ export function useBulkImportTemplateSpecimenTypes(options: { } } -/** Resolve which collections from CSV rows are missing on the server. Throws on API failure. */ +/** + * Resolve which Collections the CSV rows reference that the server does not have yet. + * Each row's name and barcode come from their own columns (see getBulkImportRowCollectionRef), + * so one row yields at most one Collection to create. Throws on API failure. + */ export async function fetchBulkImportMissingCollections(params: { rows: CSVRow[] - collectionType?: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' - getRowCollectionName?: (row: CSVRow) => string | undefined - getRowCollectionCheck?: ( - row: CSVRow - ) => { identifier: string; type: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' } | undefined + getRowCollectionRef: (row: CSVRow) => BulkImportCollectionRef | undefined }): Promise { - const { rows, collectionType, getRowCollectionName, getRowCollectionCheck } = params - - const checkData: Array<{ identifier: string; type: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' }> = - [] - const seen = new Set() + const { rows, getRowCollectionRef } = params + // One entry per Collection: rows that share a name (or, without a name, a barcode) share it. + const refs = new Map() for (const row of rows) { - if (getRowCollectionCheck) { - const check = getRowCollectionCheck(row) - if (check) { - const key = `${check.type}:${check.identifier}` - if (!seen.has(key)) { - checkData.push(check) - seen.add(key) - } - } - } else if (collectionType && getRowCollectionName) { - const collectionName = getRowCollectionName(row) - if (collectionName && !seen.has(`${collectionType}:${collectionName}`)) { - checkData.push({ identifier: collectionName, type: collectionType }) - seen.add(`${collectionType}:${collectionName}`) - } - } - if (row.collection_barcode && collectionType) { - const key = `${collectionType}:${row.collection_barcode}` - if (!seen.has(key)) { - checkData.push({ identifier: row.collection_barcode, type: collectionType }) - seen.add(key) - } + const ref = getRowCollectionRef(row) + if (!ref) continue + const key = ref.name ? `${ref.type}:name:${ref.name}` : `${ref.type}:barcode:${ref.barcode ?? ''}` + const existing = refs.get(key) + if (!existing) { + refs.set(key, ref) + } else if (!existing.barcode && ref.barcode) { + existing.barcode = ref.barcode } } + const checkData: Array<{ identifier: string; type: BulkImportCollectionType }> = [] + const checkIndex = new Map() + const addCheck = (type: BulkImportCollectionType, identifier: string) => { + const key = `${type}:${identifier}` + if (checkIndex.has(key)) return + checkIndex.set(key, checkData.length) + checkData.push({ identifier, type }) + } + for (const ref of refs.values()) { + if (ref.name) addCheck(ref.type, ref.name) + if (ref.barcode) addCheck(ref.type, ref.barcode) + } + if (checkData.length === 0) return [] - const response = await collectionsApi.check({ collections: checkData }) - const results = response.results + const { results } = await collectionsApi.check({ collections: checkData }) + const exists = (type: BulkImportCollectionType, identifier: string | undefined) => { + if (!identifier) return false + const index = checkIndex.get(`${type}:${identifier}`) + return index != null && results[index]?.exists === true + } const missing: BulkImportMissingCollection[] = [] - const found = new Set() - - for (let i = 0; i < results.length; i++) { - const result = results[i] - const check = checkData[i] - if (!result.exists) { - const isBarcode = - result.identifier.match(/^[A-Z0-9-]+$/) != null && result.identifier.length > 5 - - const dedupeKey = `${check.type}:${result.identifier}` - if (!found.has(dedupeKey)) { - missing.push({ - name: isBarcode ? '' : result.identifier, - collectionType: check.type, - barcode: isBarcode ? result.identifier : undefined, - collectionBarcode: isBarcode ? result.identifier : undefined, - locationId: null, - status: 'pending', - }) - found.add(dedupeKey) - } - } else { - found.add(`${check.type}:${result.identifier}`) - } + for (const ref of refs.values()) { + if (exists(ref.type, ref.name) || exists(ref.type, ref.barcode)) continue + missing.push({ + name: ref.name ?? '', + collectionType: ref.type, + ...(ref.barcode ? { barcode: ref.barcode } : {}), + locationId: null, + status: 'pending', + }) } - return missing } diff --git a/packages/web/src/lib/__tests__/bulk-import-csv.test.ts b/packages/web/src/lib/__tests__/bulk-import-csv.test.ts index 6b79b914..228d7d67 100644 --- a/packages/web/src/lib/__tests__/bulk-import-csv.test.ts +++ b/packages/web/src/lib/__tests__/bulk-import-csv.test.ts @@ -1,5 +1,13 @@ import { describe, it, expect } from 'vitest' +import { bulkCombinedValidateRequestSchema, csvRowToContainerWriteInput } from '@sampledb/contract' import { buildBulkImportTemplateContent } from '../bulk-import-csv' +import { buildBulkCombinedRequestPayload } from '../bulk-import-payload' +import { + getBulkImportRequiredFields, + mapBulkImportRowsToPayload, + parseBulkImportCSV, +} from '../bulk-import-validation' +import type { ContainerType } from '../container-types' describe('buildBulkImportTemplateContent', () => { it('subjects template has no specimen types', () => { @@ -81,4 +89,60 @@ describe('buildBulkImportTemplateContent', () => { expect(csvContent).toContain('bag_name') expect(csvContent).not.toContain('position') }) + + describe('every downloaded template passes validation unchanged', () => { + const containerTypes: ContainerType[] = ['micronix_tube', 'cryovial_tube', 'paper', 'static_well'] + const cases = containerTypes.flatMap((containerType) => + (['specimens', 'combined'] as const).flatMap((importType) => + [undefined, 'ST'].map((fixedStudyShortCode) => ({ containerType, importType, fixedStudyShortCode })) + ) + ) + + it.each(cases)( + '$importType template for $containerType (fixed study: $fixedStudyShortCode)', + ({ containerType, importType, fixedStudyShortCode }) => { + const { csvContent } = buildBulkImportTemplateContent({ + importType, + containerType, + fixedStudyShortCode, + specimenTypeNames: ['Whole Blood'], + }) + const rows = parseBulkImportCSV(csvContent) + expect(rows.length).toBeGreaterThan(0) + + const required = getBulkImportRequiredFields({ importType, containerType, fixedStudyShortCode }) + for (const row of rows) { + for (const field of required) { + // "box_name or bag_name": exactly one of the columns has a value. + const filled = field.split(' or ').filter((column) => (row[column] ?? '').trim() !== '') + expect(filled, `${field} in ${JSON.stringify(row)}`).toHaveLength(1) + } + // The contract's own CSV row rules, with a sheet name required for paper. + expect( + csvRowToContainerWriteInput(row, { defaultContainerType: containerType, requireSheetName: true }) + ).toMatchObject({ success: true }) + } + + const payload = buildBulkCombinedRequestPayload( + mapBulkImportRowsToPayload(rows, { importType, containerType, fixedStudyShortCode }), + { containerType, fixedStudyShortCode, missingCollections: [], atomicMode: 'full_file' } + ) + expect(bulkCombinedValidateRequestSchema.safeParse(payload).success).toBe(true) + } + ) + + it('paper example row names a box, a sheet, and a sublabel', () => { + const { csvContent } = buildBulkImportTemplateContent({ + importType: 'combined', + containerType: 'paper', + specimenTypeNames: ['Blood Spot'], + }) + expect(parseBulkImportCSV(csvContent)[0]).toMatchObject({ + box_name: 'BOX-003', + bag_name: '', + sheet_name: 'SHEET-001', + sublabel: 'SPOT-001', + }) + }) + }) }) diff --git a/packages/web/src/lib/__tests__/bulk-import-validation.test.ts b/packages/web/src/lib/__tests__/bulk-import-validation.test.ts index 94110996..58ba09b2 100644 --- a/packages/web/src/lib/__tests__/bulk-import-validation.test.ts +++ b/packages/web/src/lib/__tests__/bulk-import-validation.test.ts @@ -7,7 +7,7 @@ import { getBulkImportCollectionType, getBulkImportRowCollectionName, resolveBulkImportPaperParent, - getBulkImportRowCollectionCheck, + getBulkImportRowCollectionRef, } from '../bulk-import-validation' describe('bulk-import-validation', () => { @@ -52,6 +52,11 @@ describe('bulk-import-validation', () => { expect(fields).toContain('barcode') expect(fields).toContain('position') }) + it('lists box_name and bag_name for paper as alternatives, not both required', () => { + expect( + getBulkImportRequiredFields({ importType: 'combined', containerType: 'paper', fixedStudyShortCode: 'ST' }) + ).toEqual(['subject_name', 'specimen_type_name', 'box_name or bag_name', 'sheet_name']) + }) }) describe('getBulkImportOptionalFields', () => { @@ -177,12 +182,38 @@ describe('bulk-import-validation', () => { }) }) - describe('getBulkImportRowCollectionCheck', () => { + describe('getBulkImportRowCollectionRef', () => { it('returns bag type for paper bag rows', () => { - expect(getBulkImportRowCollectionCheck({ bag_name: 'Bag-A' }, 'paper')).toEqual({ - identifier: 'Bag-A', + expect(getBulkImportRowCollectionRef({ bag_name: 'Bag-A' }, 'paper')).toEqual({ + name: 'Bag-A', type: 'bag', }) }) + + it('reads a barcode-shaped plate_name as a name', () => { + expect(getBulkImportRowCollectionRef({ plate_name: 'PLATE-001' }, 'micronix_tube')).toEqual({ + name: 'PLATE-001', + type: 'micronix_plate', + }) + }) + + it('reads a name-shaped collection_barcode as a barcode', () => { + expect(getBulkImportRowCollectionRef({ collection_barcode: 'my box' }, 'cryovial_tube')).toEqual({ + barcode: 'my box', + type: 'cryovial_box', + }) + }) + + it('keeps the name and barcode of one row together', () => { + expect( + getBulkImportRowCollectionRef({ plate_name: 'Plate A', collection_barcode: 'PB-1' }, 'static_well') + ).toEqual({ name: 'Plate A', barcode: 'PB-1', type: 'micronix_plate' }) + }) + + it('returns undefined when the row names no Collection', () => { + expect(getBulkImportRowCollectionRef({ plate_name: ' ' }, 'micronix_tube')).toBeUndefined() + expect(getBulkImportRowCollectionRef({ box_name: 'B', bag_name: 'G' }, 'paper')).toBeUndefined() + expect(getBulkImportRowCollectionRef({ plate_name: 'P' }, 'none')).toBeUndefined() + }) }) }) diff --git a/packages/web/src/lib/bulk-import-csv.ts b/packages/web/src/lib/bulk-import-csv.ts index 6c64e5ea..4c0396b9 100644 --- a/packages/web/src/lib/bulk-import-csv.ts +++ b/packages/web/src/lib/bulk-import-csv.ts @@ -27,7 +27,8 @@ function getTemplateExampleRow( case 'cryovial_tube': return [...base, 'BOX-001', '', 'B01', ''] case 'paper': - return [...base, 'BOX-003', 'SPOT-001', ''] + // box_name, bag_name, sheet_name, sublabel, comment: a row names its box or its bag, not both. + return [...base, 'BOX-003', '', 'SHEET-001', 'SPOT-001', ''] case 'static_well': return [...base, 'PLATE-002', 'A01', ''] default: diff --git a/packages/web/src/lib/bulk-import-validation.ts b/packages/web/src/lib/bulk-import-validation.ts index 1027be77..bbc85a94 100644 --- a/packages/web/src/lib/bulk-import-validation.ts +++ b/packages/web/src/lib/bulk-import-validation.ts @@ -18,9 +18,11 @@ export interface BulkImportValidationError { export type ImportType = 'subjects' | 'specimens' | 'combined' +export type BulkImportCollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' + export function getBulkImportCollectionType( containerType: ContainerType | 'none' | '' -): 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' | null { +): BulkImportCollectionType | null { switch (containerType) { case 'micronix_tube': case 'static_well': @@ -52,7 +54,8 @@ export function getBulkImportRequiredFields(opts: { const containerFields: Record = { micronix_tube: ['plate_name', 'barcode', 'position'], cryovial_tube: ['box_name', 'position'], - paper: ['box_name', 'bag_name', 'sheet_name'], + // A paper row names its box or its bag, never both. + paper: ['box_name or bag_name', 'sheet_name'], static_well: ['plate_name', 'position'], } return [...base, ...containerFields[containerType]] @@ -137,23 +140,38 @@ export function getBulkImportRowCollectionName( return t || undefined } -/** Per-row collection check for missing-collection resolution (supports mixed box/bag paper rows). */ -export function getBulkImportRowCollectionCheck( +/** + * The Collection a row places its container in. `name` comes from the name column + * (plate_name, box_name, or bag_name) and `barcode` from collection_barcode; a value is + * never reclassified by its shape. + */ +export interface BulkImportCollectionRef { + type: BulkImportCollectionType + name?: string + barcode?: string +} + +/** Per-row Collection reference for missing-collection resolution (supports mixed box/bag paper rows). */ +export function getBulkImportRowCollectionRef( row: CSVRow, containerType: ContainerType | 'none' | '' -): { identifier: string; type: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' } | undefined { +): BulkImportCollectionRef | undefined { if (containerType === 'paper') { const parent = resolveBulkImportPaperParent(row) if (!parent || 'error' in parent) return undefined - return { - identifier: parent.collectionName, - type: parent.parentCollectionType, - } + return { type: parent.parentCollectionType, name: parent.collectionName } } - const collectionType = getBulkImportCollectionType(containerType) + const type = getBulkImportCollectionType(containerType) + if (!type) return undefined const name = getBulkImportRowCollectionName(row, containerType) - if (!collectionType || !name) return undefined - return { identifier: name, type: collectionType } + // Only plates and cryovial boxes carry barcodes. + const barcode = trimCell(row.collection_barcode) || undefined + if (!name && !barcode) return undefined + return { + type, + ...(name ? { name } : {}), + ...(barcode ? { barcode } : {}), + } } function getRowCollectionName( From 7babf1c0a126fba9795403c1fc7ee35912461717 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:33:14 -0700 Subject: [PATCH 05/54] fix(position): enforce one Grid position rule on every Container write (#102) Add parseGridPosition and GRID_GEOMETRY to @sampledb/contract: a row letter plus a column number, normalized to A01 form and checked against the Collection type (micronix plate A01-H12, cryovial box A01-J10). Moves and Scan moves, Specimen registration (bulk import, specimens, derivations, control batches) and the derivation CSV preview now reject any other position with an error naming the row and the value; the cryovial Scan move flags bad target positions at upload. Lookups keep the lenient normalizer so legacy malformed rows still resolve and move. Closes #102 Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 2 +- .../src/lib/__tests__/container-move.test.ts | 114 +++++++++++++++++- .../src/lib/__tests__/derivations-csv.test.ts | 24 ++++ packages/api/src/lib/container-move.ts | 18 +-- packages/api/src/lib/derivations-csv.ts | 15 ++- packages/api/src/lib/normalize-position.ts | 6 +- .../__tests__/specimen-registration.test.ts | 45 +++++++ .../api/src/lib/specimen-registration/plan.ts | 7 +- .../src/routes/__tests__/collections.test.ts | 44 +++++++ .../src/routes/__tests__/containers.test.ts | 13 ++ .../api/src/routes/__tests__/imports.test.ts | 71 ++++++++++- .../src/__tests__/grid-position.test.ts | 57 +++++++++ packages/contract/src/grid-position.ts | 73 +++++++++++ packages/contract/src/index.ts | 8 ++ .../bulk-operations/container-movement.md | 3 +- .../docs/guides/bulk-operations/import.md | 13 +- .../web/src/lib/micronix-plate-positions.ts | 27 ++--- .../lib/scan-move/__tests__/ingest.test.ts | 14 +++ packages/web/src/lib/scan-move/variants.ts | 11 +- 19 files changed, 524 insertions(+), 41 deletions(-) create mode 100644 packages/contract/src/__tests__/grid-position.test.ts create mode 100644 packages/contract/src/grid-position.ts diff --git a/CONTEXT.md b/CONTEXT.md index 99674c4f..5f8638b2 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -132,7 +132,7 @@ Where a **Container** sits in the physical storage hierarchy — its **grid posi _Avoid_: Enrichment (implementation term); conflating placement with specimen or source lookups; placing paper directly on a box or bag without a sheet **Grid position**: -The well or slot of a tube or well **Container** in its plate or box (e.g. A01). Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. +The well or slot of a tube or well **Container** in its plate or box (e.g. A01): a row letter plus a two-digit column, inside the **Collection**'s geometry (micronix plate A01–H12, cryovial box A01–J10). Every write that sets one normalizes `a1` to `A01` and rejects anything else. Legacy rows may still hold malformed positions. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. _Avoid_: Well (as the domain term); treating a missing position as occupying a slot; requiring every tube to have a position before it can exist in a Collection ### Export diff --git a/packages/api/src/lib/__tests__/container-move.test.ts b/packages/api/src/lib/__tests__/container-move.test.ts index be6fff0a..838d5a2b 100644 --- a/packages/api/src/lib/__tests__/container-move.test.ts +++ b/packages/api/src/lib/__tests__/container-move.test.ts @@ -4,6 +4,8 @@ import { createTestStorageType, createTestLocation, createTestMicronixPlate, + createTestCryovialBox, + createTestCryovialTube, createTestSpecimenType, createTestSpecimen, createTestUnit, @@ -21,7 +23,7 @@ import { type ContainerInfo, } from '../container-move' import { storageContainer } from '../../db/schema' -import { micronixTube, staticWell, paper } from '../../db/schema' +import { micronixTube, cryovialTube, staticWell, paper } from '../../db/schema' import type { Database } from '../../db/client' import { utcNow } from '../datetime' @@ -555,6 +557,76 @@ describe('container-move', () => { expect(tubes.find((t) => t.barcode === 'MT-T9')?.position).toBe('D01') }) + it.each([ + { target: 'Z99', reason: 'outside' }, + { target: 'I01', reason: 'outside' }, + { target: 'A13', reason: 'outside' }, + { target: 'hello world', reason: 'not a grid position' }, + { target: '--A01', reason: 'not a grid position' }, + ])('rejects a move to $target on a plate and writes nothing', async ({ target, reason }) => { + const { plate, addTube } = await setupPlate() + await addTube('MT-OK', 'A01') + await addTube('MT-BAD', 'A02') + + const result = await executeMoves(testDb, { + mappings: [{ fromCollectionName: plate.name, toCollectionName: plate.name }], + moves: [ + { identifier: { type: 'barcode', barcode: 'MT-OK' }, targetPosition: 'B01' }, + { identifier: { type: 'barcode', barcode: 'MT-BAD' }, targetPosition: target }, + ], + }) + + expect(result.success).toBe(false) + expect(result.moved).toBe(0) + expect(result.errors).toEqual([{ row: 2, error: expect.stringContaining(`"${target}"`) }]) + expect(result.errors?.[0]?.error).toContain(reason) + const tubes = await testDb.select().from(micronixTube).all() + expect(tubes.map((t) => [t.barcode, t.position])).toEqual([['MT-OK', 'A01'], ['MT-BAD', 'A02']]) + }) + + it('best_effort moves the valid rows and skips an out-of-range one', async () => { + const { plate, addTube } = await setupPlate() + await addTube('MT-OK', 'A01') + await addTube('MT-BAD', 'A02') + + const result = await executeMoves(testDb, { + atomicMode: 'best_effort', + mappings: [{ fromCollectionName: plate.name, toCollectionName: plate.name }], + moves: [ + { identifier: { type: 'barcode', barcode: 'MT-OK' }, targetPosition: 'b1' }, + { identifier: { type: 'barcode', barcode: 'MT-BAD' }, targetPosition: 'H13' }, + ], + }) + + expect(result.moved).toBe(1) + expect(result.errors).toEqual([{ row: 2, error: expect.stringContaining('"H13" is outside a micronix plate') }]) + const tubes = await testDb.select().from(micronixTube).all() + expect(tubes.map((t) => [t.barcode, t.position])).toEqual([['MT-OK', 'B01'], ['MT-BAD', 'A02']]) + }) + + it('moves on a plate that holds legacy malformed positions, including a legacy tube itself', async () => { + const { plate, addTube } = await setupPlate() + await addTube('MT-LEGACY', 'A1145854300') + await addTube('MT-DASH', '--A01') + await addTube('MT-NEW', 'B01') + + const result = await executeMoves(testDb, { + mappings: [{ fromCollectionName: plate.name, toCollectionName: plate.name }], + moves: [ + { identifier: { type: 'barcode', barcode: 'MT-NEW' }, targetPosition: 'A01' }, + { identifier: { type: 'barcode', barcode: 'MT-LEGACY' }, targetPosition: 'c1' }, + ], + }) + + expect(result).toEqual({ success: true, moved: 2 }) + const tubes = await testDb.select().from(micronixTube).all() + expect(Object.fromEntries(tubes.map((t) => [t.barcode, t.position]))).toEqual({ + 'MT-LEGACY': 'C01', + 'MT-DASH': '--A01', + 'MT-NEW': 'A01', + }) + }) + it('resolves a static well by container_id', async () => { const { plate, addWell } = await setupPlate() const wellId = await addWell('A01') @@ -565,6 +637,46 @@ describe('container-move', () => { }) }) + describe('cryovial box grid positions', () => { + async function setupBox() { + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id) }) + const box = await createTestCryovialBox(testDb, { name: 'CryoBox', locationId: location.id }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const specimen = await createTestSpecimen(testDb, specimenType.id) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const container = await createTestStorageContainer(testDb, { specimenId: specimen.id, unitId: unit.id }) + await createTestCryovialTube(testDb, { containerId: container.id, collectionId: box.id, barcode: 'CV-1', position: 'A01' }) + return box + } + + it.each([ + { target: 'j10', stored: 'J10' }, + { target: 'I9', stored: 'I09' }, + ])('stores $target in a box as $stored', async ({ target, stored }) => { + const box = await setupBox() + const result = await executeMoves(testDb, { + mappings: [{ fromCollectionName: box.name, toCollectionName: box.name }], + moves: [{ identifier: { type: 'barcode', barcode: 'CV-1' }, targetPosition: target }], + }) + expect(result).toEqual({ success: true, moved: 1 }) + const [tube] = await testDb.select().from(cryovialTube).all() + expect(tube.position).toBe(stored) + }) + + it.each(['K01', 'A11', 'H12'])('rejects %s, outside a box', async (target) => { + const box = await setupBox() + const result = await executeMoves(testDb, { + mappings: [{ fromCollectionName: box.name, toCollectionName: box.name }], + moves: [{ identifier: { type: 'barcode', barcode: 'CV-1' }, targetPosition: target }], + }) + expect(result.success).toBe(false) + expect(result.errors).toEqual([{ row: 1, error: `Position "${target}" is outside a cryovial box (A01–J10)` }]) + const [tube] = await testDb.select().from(cryovialTube).all() + expect(tube.position).toBe('A01') + }) + }) + describe('paper moves between same-named sheets', () => { it('moves papers from Sheet-1 of one box to Sheet-1 of another using parent names', async () => { const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) diff --git a/packages/api/src/lib/__tests__/derivations-csv.test.ts b/packages/api/src/lib/__tests__/derivations-csv.test.ts index 5aef2c8a..378f8c92 100644 --- a/packages/api/src/lib/__tests__/derivations-csv.test.ts +++ b/packages/api/src/lib/__tests__/derivations-csv.test.ts @@ -361,6 +361,30 @@ describe('derivations-csv', () => { expect(result.rows[2].error).toMatch(/more than once in your file/) }) + it('rejects an out-of-range or garbage position in validation', async () => { + const { parent } = await seedParent() + const csv = [header, `${parent.id},PlateDef,b1,C1`, `${parent.id},PlateDef,Z99,C2`, `${parent.id},PlateDef,hello world,C3`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, settings) + + expect(result.rows.map((r) => r.valid)).toEqual([true, false, false]) + expect(result.rows[1].error).toBe('Position "Z99" is outside a micronix plate (A01–H12)') + expect(result.rows[2].error).toContain('Position "hello world" is not a grid position') + }) + + it('imports a1 as A01 and refuses an out-of-range position', async () => { + const { parent } = await seedParent() + + const bad = await importDerivationsFromCsv(testDb, [header, `${parent.id},PlateDef,I01,C1`].join('\n'), { settings }) + expect(bad.rows[0]?.success).toBe(false) + expect(bad.rows[0]?.error).toContain('Position "I01" is outside a micronix plate') + + const good = await importDerivationsFromCsv(testDb, [header, `${parent.id},PlateDef,b1,C1`].join('\n'), { settings }) + expect(good.rows[0]?.success).toBe(true) + const child = await testDb.select().from(micronixTube).where(eq(micronixTube.barcode, 'C1')).get() + expect(child?.position).toBe('B01') + }) + it('warns about a mismatched source type even when the first row has no parent', async () => { const { parent, plate, unit } = await seedParent() const plasmaType = await createTestSpecimenType(testDb, { name: 'Plasma' }) diff --git a/packages/api/src/lib/container-move.ts b/packages/api/src/lib/container-move.ts index 15138b9d..2e786cc2 100644 --- a/packages/api/src/lib/container-move.ts +++ b/packages/api/src/lib/container-move.ts @@ -14,6 +14,7 @@ import { eq, and, isNull } from 'drizzle-orm' import { resolveCollection, resolveSheetsByName, type CollectionType } from './collections/collection-resolve' import { checkGridPositionOccupancy } from './container-occupancy' import { normalizePosition } from './normalize-position' +import { parseGridPosition } from '@sampledb/contract' import type { ContainerSubtype } from './container-placement' /** Staging prefix for position updates so swaps on one plate do not violate UNIQUE(collection_id, position) mid-transaction. */ @@ -328,12 +329,7 @@ export function inferCollectionTypeFromContainers(containers: ContainerInfo[]): export async function executeMoves(database: Database, request: BatchMoveRequest): Promise { try { - const { mappings, atomicMode = 'all_or_nothing' } = request - // Normalize target positions ("b1" -> "B01") so writes, conflict keys, and occupancy checks agree. - const moves = request.moves.map((move) => ({ - ...move, - targetPosition: normalizePosition(move.targetPosition) ?? '', - })) + const { mappings, moves, atomicMode = 'all_or_nothing' } = request const errors: ValidationError[] = [] const resolvedMoves = await Promise.all(moves.map(async (move, index) => ({ @@ -374,9 +370,17 @@ export async function executeMoves(database: Database, request: BatchMoveRequest for (const m of validResolvedMoves) { const requiresPosition = m.info.containerType === 'micronix_tube' || m.info.containerType === 'cryovial_tube' || m.info.containerType === 'static_well' - if (requiresPosition && (!m.move.targetPosition || typeof m.move.targetPosition !== 'string' || m.move.targetPosition.trim() === '')) { + if (!requiresPosition) continue + if (!m.move.targetPosition || typeof m.move.targetPosition !== 'string' || m.move.targetPosition.trim() === '') { errors.push({ row: m.row, error: `target_position is required for ${m.info.containerType}` }) + continue } + if (collectionType !== 'micronix_plate' && collectionType !== 'cryovial_box') continue + // Grid position rule: store "b1" as "B01" so writes, conflict keys, and occupancy checks + // agree, and reject a position the plate or box does not have. + const parsed = parseGridPosition(m.move.targetPosition, collectionType) + if (parsed.ok) m.move = { ...m.move, targetPosition: parsed.position } + else errors.push({ row: m.row, error: parsed.error }) } if (atomicMode === 'all_or_nothing' && errors.length > 0) { diff --git a/packages/api/src/lib/derivations-csv.ts b/packages/api/src/lib/derivations-csv.ts index 119a86e1..1ec3e61f 100644 --- a/packages/api/src/lib/derivations-csv.ts +++ b/packages/api/src/lib/derivations-csv.ts @@ -16,6 +16,7 @@ import { and, eq, sql } from 'drizzle-orm' import { csvRowToContainerWriteInput, parseCsv as parseCsvCells, + parseGridPosition, PAPER_DERIVATION_USE_SUBLABEL_NOT_CONTAINER_BARCODE, PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type ContainerCsvRow, @@ -23,7 +24,6 @@ import { import { createDerivation, type CreateDerivationInput } from './derivations' import { withWriteTransaction } from '../db/write-transaction' import { resolveParentContainerId } from './derivations/parent-container-resolver' -import { normalizePosition } from './normalize-position' import { checkGridPositionOccupancy } from './container-occupancy' import { CollectionPlanner } from './specimen-registration/collections' @@ -528,14 +528,21 @@ export async function validateDerivationsCsv( // For tube types, validate position and check for duplicate (existing collection or within CSV) if (containerType === 'micronix_tube' || containerType === 'cryovial_tube') { - // Normalize ("A2" -> "A02") so checks match what the import stores. - const position = normalizePosition(row.position) ?? '' - if (!position) { + if (!row.position?.trim()) { validationRow.error = 'position is required for each row when deriving to a plate or box' validationRows.push(validationRow) invalidCount++ continue } + // Grid position rule: normalize ("A2" -> "A02") so checks match what the import stores. + const parsedPosition = parseGridPosition(row.position, containerType === 'micronix_tube' ? 'micronix_plate' : 'cryovial_box') + if (!parsedPosition.ok) { + validationRow.error = parsedPosition.error + validationRows.push(validationRow) + invalidCount++ + continue + } + const position = parsedPosition.position const collectionKey = `${collectionName || ''}_${row.collection_barcode || ''}_${containerType}` const positionKey = `${collectionKey}\t${position}` diff --git a/packages/api/src/lib/normalize-position.ts b/packages/api/src/lib/normalize-position.ts index 60655b26..ba8c53ee 100644 --- a/packages/api/src/lib/normalize-position.ts +++ b/packages/api/src/lib/normalize-position.ts @@ -1,4 +1,8 @@ -/** Normalize position string to match frontend format (e.g., "B1" -> "B01"). */ +/** + * Normalize a position for a lookup against stored rows ("B1" -> "B01"). Passes anything + * else through unchanged so a legacy position (e.g. "--A01") still finds its row. Not for + * writes: a write checks the position with `parseGridPosition` from `@sampledb/contract`. + */ export function normalizePosition(position: string | null | undefined): string | null { if (!position || !position.trim()) return null diff --git a/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts b/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts index 34100513..7e49bc5e 100644 --- a/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts +++ b/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts @@ -11,9 +11,12 @@ import { createTestUnit, createTestSpecimenTypeContainerType, createTestContainerTypeUnit, + createTestStorageContainer, + createTestMicronixTube, } from '../../../__tests__/helpers/factories' import type { Database } from '../../../db/client' import { + cryovialTube, micronixPlate, micronixTube, paper, @@ -310,6 +313,48 @@ describe('Specimen registration', () => { ]) }) + describe('Grid position rule', () => { + it.each([ + { position: 'Z99', message: 'Position "Z99" is outside a micronix plate (A01–H12)' }, + { position: 'A13', message: 'Position "A13" is outside a micronix plate (A01–H12)' }, + { position: 'I01', message: 'Position "I01" is outside a micronix plate (A01–H12)' }, + { position: 'hello world', message: expect.stringContaining('Position "hello world" is not a grid position') }, + { position: 'NA01', message: expect.stringContaining('Position "NA01" is not a grid position') }, + ])('rejects $position on a plate and writes nothing', async ({ position, message }) => { + const error = await rejection([entry([tube('T1', { name: 'P1', position })])]) + + expect(error.errors).toEqual([{ entryIndex: 0, containerIndex: 0, field: 'position', message }]) + expect(await count(specimen)).toBe(0) + expect(await count(storageContainer)).toBe(0) + }) + + it('accepts J10 in a cryovial box and rejects K01', async () => { + const box = (barcode: string, position: string): RegistrationContainer => ({ + containerType: 'cryovial_tube', + barcode, + collection: { type: 'cryovial_box', name: 'BOX', locationId, position }, + }) + const error = await rejection([entry([box('C1', 'j10'), box('C2', 'K01')])]) + expect(error.errors).toEqual([ + { entryIndex: 0, containerIndex: 1, field: 'position', message: 'Position "K01" is outside a cryovial box (A01–J10)' }, + ]) + + await registerSpecimens(db, [entry([box('C1', 'j10')])]) + const [row] = await db.select().from(cryovialTube) + expect(row?.position).toBe('J10') + }) + + it('registers into a plate that holds a legacy malformed position', async () => { + const container = await createTestStorageContainer(db, { unitId }) + await createTestMicronixTube(db, { containerId: container.id, collectionId: plate.id, barcode: 'LEGACY', position: '--A01' }) + + await registerSpecimens(db, [entry([tube('T1', { name: 'P1', position: 'A1' })])]) + + const rows = await db.select().from(micronixTube) + expect(Object.fromEntries(rows.map((r) => [r.barcode, r.position]))).toEqual({ LEGACY: '--A01', T1: 'A01' }) + }) + }) + it('rejects a unit not allowed for the container type', async () => { const other = await createTestUnit(db, { symbol: 'mg', name: 'milligram', category: 'mass' }) diff --git a/packages/api/src/lib/specimen-registration/plan.ts b/packages/api/src/lib/specimen-registration/plan.ts index 5f9e07e2..542f96db 100644 --- a/packages/api/src/lib/specimen-registration/plan.ts +++ b/packages/api/src/lib/specimen-registration/plan.ts @@ -13,7 +13,7 @@ import { type Specimen, } from '../../db/schema' import { getDefaultRemainingQuantity, getDefaultTotalQuantity, getDefaultUnit } from '../defaults' -import { normalizePosition } from '../normalize-position' +import { parseGridPosition } from '@sampledb/contract' import { checkGridPositionOccupancy, occupiedGridPositionMessage } from '../container-occupancy' import { validateCollectionDate, @@ -284,7 +284,10 @@ async function planContainer( problems.push({ field: 'collection', message: resolved.error }) } else { collection = resolved.target - position = normalizePosition(input.collection?.position) + const rawPosition = input.collection?.position?.trim() + const parsed = rawPosition ? parseGridPosition(rawPosition, gridType) : null + if (parsed && !parsed.ok) problems.push({ field: 'position', message: parsed.error }) + position = parsed?.ok ? parsed.position : null if (position) { if ('existingId' in collection) { const occupancy = await checkGridPositionOccupancy(database, { diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 7a80c107..67d286fb 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -17,6 +17,8 @@ import { createTestStudySubject, createTestBox, createTestSheet, + createTestStorageContainer, + createTestMicronixTube, } from '../../__tests__/helpers/factories' import { setScannerConfigurations } from '../../lib/settings' import { eq } from 'drizzle-orm' @@ -161,6 +163,48 @@ describe('Collections API', () => { expect(res.status).toBe(400) }) + it.each(['Z99', 'hello world'])('returns 400 naming the row for a move to %s and writes nothing', async (target) => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'P96', locationId: loc.id }) + const container = await createTestStorageContainer(ctx.db) + await createTestMicronixTube(ctx.db, { containerId: container.id, collectionId: plate.id, barcode: 'MT-1', position: 'A01' }) + + const res = await ctx.request('/api/collections/containers/move', { + method: 'POST', + json: { + mappings: [{ fromCollectionName: 'P96', toCollectionName: 'P96' }], + moves: [{ identifier: { type: 'barcode', barcode: 'MT-1' }, targetPosition: target }], + }, + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.errors).toEqual([{ row: 1, error: expect.stringContaining(`Position "${target}"`) }]) + const tube = ctx.db.select().from(micronixTube).where(eq(micronixTube.id, container.id)).get() + expect(tube?.position).toBe('A01') + }) + + it('stores a1 as A01', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'P96', locationId: loc.id }) + const container = await createTestStorageContainer(ctx.db) + await createTestMicronixTube(ctx.db, { containerId: container.id, collectionId: plate.id, barcode: 'MT-1', position: 'B05' }) + + const res = await ctx.request('/api/collections/containers/move', { + method: 'POST', + json: { + mappings: [{ fromCollectionName: 'P96', toCollectionName: 'P96' }], + moves: [{ identifier: { type: 'barcode', barcode: 'MT-1' }, targetPosition: 'a1' }], + }, + }) + + expect(res.status).toBe(200) + const tube = ctx.db.select().from(micronixTube).where(eq(micronixTube.id, container.id)).get() + expect(tube?.position).toBe('A01') + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/collections/containers/move', { method: 'POST', diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index c56dd06a..56b639d4 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -294,6 +294,19 @@ describe('Containers API', () => { expect(row?.comment).toBeNull() }) + it.each(['Z99', 'hello world', 'b1'])('never writes a grid position, even %s', async (position) => { + // Container edit sets no Grid position; a move changes it under the Grid position rule. + const boxRecord = await createTestCryovialBox(ctx.db, { name: `BOX-POS-${Date.now()}`, locationId: testLocation.id }) + const container = await createTestStorageContainer(ctx.db, { unitId: testUnit.id }) + await createTestCryovialTube(ctx.db, { containerId: container.id, collectionId: boxRecord.id, barcode: 'POS-1', position: 'A01' }) + + const res = await ctx.request(`/api/containers/${container.id}`, { method: 'PATCH', json: { comment: 'moved?', position } }) + + expect(res.status).toBe(200) + const row = await ctx.db.select().from(cryovialTube).where(eq(cryovialTube.id, container.id)).get() + expect(row?.position).toBe('A01') + }) + it('returns 400 when cryovial_tube barcode is already in use', async () => { const now = utcNow() const [boxRecord] = await ctx.db diff --git a/packages/api/src/routes/__tests__/imports.test.ts b/packages/api/src/routes/__tests__/imports.test.ts index 48c5cb05..c7d64927 100644 --- a/packages/api/src/routes/__tests__/imports.test.ts +++ b/packages/api/src/routes/__tests__/imports.test.ts @@ -12,7 +12,7 @@ import { createTestStorageType, createTestUnit, } from '../../__tests__/helpers/factories' -import { studySubject, specimen, specimenTypeContainerType, containerTypeUnit, micronixPlate, cryovialBox, settings, storageContainer } from '../../db/schema' +import { studySubject, specimen, specimenTypeContainerType, containerTypeUnit, micronixPlate, micronixTube, cryovialBox, settings, storageContainer } from '../../db/schema' import { setContainerDefaults } from '../../lib/settings' import { eq } from 'drizzle-orm' import { failInsertsWithUniqueViolation } from '../../__tests__/helpers/constraint-race' @@ -495,6 +495,75 @@ describe('Imports API', () => { expect(data.errors.some((e) => e.message.includes('cannot contain collections'))).toBe(true) }) + describe('Grid position rule', () => { + async function setupPlateImport() { + await createTestStudy(ctx.db, { title: 'Grid Study', shortCode: 'GRID' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer', description: 'Test' }) + const loc = await createTestLocation(ctx.db, { + name: 'Shelf', + parentId: null, + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await ctx.db.insert(specimenTypeContainerType).values({ specimenTypeId: specimenType.id, containerType: 'micronix_tube' }) + await ctx.db.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + await setContainerDefaults(ctx.db, { micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' } }) + const row = (rowIndex: number, barcode: string, position: string) => ({ + subjectName: `S${rowIndex}`, + specimens: [ + { + rowIndex, + specimenTypeName: 'Blood', + collectionDate: '2025-01-01', + container: { + containerType: 'micronix_tube', + barcode, + collection: { type: 'micronix_plate', name: 'GRID-P1', locationId: loc.id, position }, + }, + }, + ], + }) + return { row } + } + + it.each([ + { position: 'Z99', message: 'Position "Z99" is outside a micronix plate (A01–H12)' }, + { position: 'A13', message: 'Position "A13" is outside a micronix plate (A01–H12)' }, + { position: 'hello world', message: expect.stringContaining('Position "hello world" is not a grid position') }, + ])('fails validation of $position with a row-numbered error', async ({ position, message }) => { + const { row } = await setupPlateImport() + const res = await ctx.request('/api/imports/bulk-combined/validate', { + method: 'POST', + json: { studyShortCode: 'GRID', atomicMode: 'full_file', subjects: [row(1, 'BC-1', 'a1'), row(2, 'BC-2', position)] }, + }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { valid: boolean; errors: Array<{ rowIndex?: number; message: string }> } + expect(data.valid).toBe(false) + expect(data.errors).toEqual([expect.objectContaining({ rowIndex: 2, message })]) + }) + + it('rejects an out-of-range position on import with 400 and stores a1 as A01 otherwise', async () => { + const { row } = await setupPlateImport() + const bad = await ctx.request('/api/imports/bulk-combined', { + method: 'POST', + json: { studyShortCode: 'GRID', atomicMode: 'full_file', subjects: [row(1, 'BC-1', 'a1'), row(2, 'BC-2', 'Z99')] }, + }) + expect(bad.status).toBe(400) + expect(await ctx.db.select().from(micronixTube)).toHaveLength(0) + + const good = await ctx.request('/api/imports/bulk-combined', { + method: 'POST', + json: { studyShortCode: 'GRID', atomicMode: 'full_file', subjects: [row(1, 'BC-1', 'a1'), row(2, 'BC-2', 'A2')] }, + }) + expect(good.status).toBe(201) + const tubes = await ctx.db.select().from(micronixTube) + expect(Object.fromEntries(tubes.map((t) => [t.barcode, t.position]))).toEqual({ 'BC-1': 'A01', 'BC-2': 'A02' }) + }) + }) + it('returns valid: false when collection date is invalid (future)', async () => { await createTestStudy(ctx.db, { title: 'Date Study', shortCode: 'DATED' }) await createTestSpecimenType(ctx.db, { name: 'Blood' }) diff --git a/packages/contract/src/__tests__/grid-position.test.ts b/packages/contract/src/__tests__/grid-position.test.ts new file mode 100644 index 00000000..c7ef5722 --- /dev/null +++ b/packages/contract/src/__tests__/grid-position.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it } from 'bun:test' +import { gridPositionRange, parseGridPosition, type GridCollectionType } from '../grid-position' + +describe('parseGridPosition', () => { + const normalized: Array<{ raw: string; type: GridCollectionType; expected: string }> = [ + { raw: 'A1', type: 'micronix_plate', expected: 'A01' }, + { raw: 'a1', type: 'micronix_plate', expected: 'A01' }, + { raw: 'a01', type: 'micronix_plate', expected: 'A01' }, + { raw: ' b7 ', type: 'micronix_plate', expected: 'B07' }, + { raw: 'H12', type: 'micronix_plate', expected: 'H12' }, + { raw: 'J10', type: 'cryovial_box', expected: 'J10' }, + { raw: 'i9', type: 'cryovial_box', expected: 'I09' }, + ] + for (const { raw, type, expected } of normalized) { + it(`stores "${raw}" on a ${type} as ${expected}`, () => { + expect(parseGridPosition(raw, type)).toEqual({ ok: true, position: expected }) + }) + } + + const outOfRange: Array<{ raw: string; type: GridCollectionType }> = [ + { raw: 'Z99', type: 'micronix_plate' }, + { raw: 'I01', type: 'micronix_plate' }, + { raw: 'A13', type: 'micronix_plate' }, + { raw: 'A00', type: 'micronix_plate' }, + { raw: 'K01', type: 'cryovial_box' }, + { raw: 'A11', type: 'cryovial_box' }, + ] + for (const { raw, type } of outOfRange) { + it(`rejects "${raw}" as outside a ${type}`, () => { + const result = parseGridPosition(raw, type) + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.error).toContain(`"${raw}"`) + expect(result.error).toContain('outside') + expect(result.error).toContain(gridPositionRange(type)) + } + }) + } + + const garbage = ['B012', 'hello world', '--A01', 'NA01', 'A1145854300', 'AA01', '1A', 'A', 'A-1', 'A 1', '__mv_12'] + for (const raw of garbage) { + it(`rejects "${raw}" as not a grid position`, () => { + const result = parseGridPosition(raw, 'micronix_plate') + expect(result.ok).toBe(false) + if (!result.ok) expect(result.error).toContain(`"${raw}"`) + }) + } + + it('rejects a blank position', () => { + expect(parseGridPosition(' ', 'cryovial_box')).toEqual({ ok: false, error: 'Position is required for a cryovial box (A01–J10)' }) + }) + + it('names each range', () => { + expect(gridPositionRange('micronix_plate')).toBe('A01–H12') + expect(gridPositionRange('cryovial_box')).toBe('A01–J10') + }) +}) diff --git a/packages/contract/src/grid-position.ts b/packages/contract/src/grid-position.ts new file mode 100644 index 00000000..9ba443c3 --- /dev/null +++ b/packages/contract/src/grid-position.ts @@ -0,0 +1,73 @@ +/** + * The Grid position rule: which wells or slots a plate or box has, and how a typed position + * maps onto one. Every Container write that sets a Grid position (moves, Scan moves, bulk + * import, Specimen registration, derivations) checks the position with `parseGridPosition`. + */ + +/** A Collection whose Containers sit at Grid positions. */ +export type GridCollectionType = 'micronix_plate' | 'cryovial_box' + +export interface GridGeometry { + /** Row count; rows are lettered from A. */ + rows: number + /** Column count; columns are numbered from 1. */ + columns: number +} + +/** + * Plate and box dimensions. SampleDB stores no size per Collection, so each type has one + * geometry: micronix plates are 96-well (8×12). Cryovial boxes in use are 9×9 and 10×10, so a + * box allows the larger, A–J by 1–10. + */ +export const GRID_GEOMETRY: Readonly>> = { + micronix_plate: { rows: 8, columns: 12 }, + cryovial_box: { rows: 10, columns: 10 }, +} + +export type GridPositionResult = { ok: true; position: string } | { ok: false; error: string } + +const COLLECTION_LABEL: Record = { + micronix_plate: 'micronix plate', + cryovial_box: 'cryovial box', +} + +const POSITION_PATTERN = /^([A-Za-z])(\d{1,2})$/ + +/** The range of a Collection type's Grid positions, as in "A01–H12". */ +export function gridPositionRange(collectionType: GridCollectionType): string { + const { rows, columns } = GRID_GEOMETRY[collectionType] + return `A01–${formatGridPosition(rows - 1, columns)}` +} + +function formatGridPosition(rowIndex: number, column: number): string { + return `${String.fromCharCode(65 + rowIndex)}${String(column).padStart(2, '0')}` +} + +/** + * Parse a typed Grid position for a Collection type. Accepts a row letter and a column number + * in either case, with or without a leading zero (`a1`, `A1`, `A01`), and returns it in + * stored form (`A01`). Rejects anything else, including a position outside the plate or box, + * with an error that names the value. + */ +export function parseGridPosition(raw: string, collectionType: GridCollectionType): GridPositionResult { + const value = raw.trim() + const label = COLLECTION_LABEL[collectionType] + const range = gridPositionRange(collectionType) + if (value === '') return { ok: false, error: `Position is required for a ${label} (${range})` } + + const match = POSITION_PATTERN.exec(value) + if (!match) { + return { + ok: false, + error: `Position "${value}" is not a grid position: use a row letter and a column number, such as A01 (${label}: ${range})`, + } + } + + const rowIndex = match[1]!.toUpperCase().charCodeAt(0) - 65 + const column = parseInt(match[2]!, 10) + const { rows, columns } = GRID_GEOMETRY[collectionType] + if (rowIndex >= rows || column < 1 || column > columns) { + return { ok: false, error: `Position "${value}" is outside a ${label} (${range})` } + } + return { ok: true, position: formatGridPosition(rowIndex, column) } +} diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1a9f103e..453be5d2 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -81,3 +81,11 @@ export { PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type PaperInboundWriteFields, } from './paper-container-inbound' +export { + GRID_GEOMETRY, + gridPositionRange, + parseGridPosition, + type GridCollectionType, + type GridGeometry, + type GridPositionResult, +} from './grid-position' diff --git a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md index 8f366eeb..b7ae927b 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md @@ -72,7 +72,7 @@ Each source plate can map to only one destination across all files. Open **Operations → Move Containers → Move Cryovial Tubes**. Command palette: **Move Cryovial Containers**. -CSV columns: `source_collection_name`, `source_position`, `target_position`. There is no destination box column. Destination comes from the file name or the **Destination Box** picker. Click **Download Template** for headers. Position format must match the box layout, for example `A5` or `25`, and stay consistent within a box. +CSV columns: `source_collection_name`, `source_position`, `target_position`. There is no destination box column. Destination comes from the file name or the **Destination Box** picker. Click **Download Template** for headers. `target_position` is a row letter A-J plus a column 1-10, for example `A5` (stored as `A05`). Upload flags any other target position by row. `source_position` is matched as stored, so a tube with an older, malformed position can still be moved to a valid one. A move to an occupied target fails unless the occupying tube is also moved in the same batch. There is no overwrite option. @@ -102,6 +102,7 @@ After execution, micronix and cryovial results are per file. Paper shows sheet c - Unresolved micronix barcodes: they are listed on Resolve. Click **Back**, fix the CSV, then **Next: Resolve Containers**. You do not need to recreate destination plates that already exist. - Relocation error on Upload (tube at an empty well not relocated): every tube currently on the destination plate must stay in place (barcode in that well) or appear at another well in the same batch. - Occupied destination at execute: adjust positions or use a different atomicity mode. +- Invalid target position at execute: the move is refused with the row and the value, for example `Position "Z99" is outside a micronix plate (A01–H12)`. Under all-or-nothing nothing moves. - Source collection conflict: the same source collection appears in multiple files with different destinations. Split the operation or consolidate files that share a destination. To move whole plates or boxes to a new freezer, see [Move collections](/docs/guides/features/collection-move/). diff --git a/packages/docs/src/content/docs/guides/bulk-operations/import.md b/packages/docs/src/content/docs/guides/bulk-operations/import.md index 74b58a6f..9f3a3a43 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/import.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/import.md @@ -74,15 +74,20 @@ Column headers are case-insensitive. `Position`, `position`, `well_position`, an **With containers:** the same columns, plus container fields for the selected type. Optional `comment` is stored on each container. Collection identifier must use the column for that type (`plate_name`, `box_name`, or `bag_name`) so it matches the templates. - Micronix: `plate_name`, unique barcode, `position` (letter A-H plus two digits, for example `A01`). -- Cryovial: `box_name`, `position`. Barcode is optional. +- Cryovial: `box_name`, `position` (letter A-J plus a column 1-10, for example `A05`). Barcode is optional. - Paper: exactly one of `box_name` or `bag_name`, plus `sheet_name`. Optional `sublabel` for the spot. - Static wells: `plate_name` and `position`, same A01-H12 format as micronix. ## Position format -For micronix and static wells, the stored form is a letter plus two zero-padded digits: `A01`. `A1` and `a1` are accepted and padded to `A01`. `B012` is not valid. +A position is a row letter plus a column number. The stored form pads the column to two digits: `A1`, `a1`, and `a01` are all stored as `A01`. `B012`, `25`, and anything else that is not a letter plus a number is rejected. -For cryovial tubes, match the box layout. Letter-number (`A5`) and numbered (`25`) positions are both accepted. Keep the format consistent within a box. +The position must also fit the plate or box: + +- Micronix plates and static wells: rows A-H, columns 1-12 (`A01`-`H12`). +- Cryovial boxes: rows A-J, columns 1-10 (`A01`-`J10`). This covers both 9×9 and 10×10 boxes. + +Validation names the row and the value of each position that breaks these rules, for example `Position "Z99" is outside a micronix plate (A01–H12)`. ## Multiple collection dates @@ -97,7 +102,7 @@ In a longitudinal study, use one CSV row per collection. Each row creates a sepa - **Location cannot contain collections.** Pick a location that is allowed to hold collections, such as a freezer or shelf configured for that. - **Position already used / duplicate position.** That well is taken in the plate or box, or the same position appears twice in the file for the same collection. - **Barcode already exists / duplicate barcode.** The barcode is already in the database or appears twice in the file. -- **Position format errors.** Use `A01` for micronix and static wells. Match the box layout for cryovial tubes. +- **Position format errors.** Use a row letter plus a column number inside the plate or box: `A01`-`H12` for micronix and static wells, `A01`-`J10` for cryovial tubes. - **Invalid specimen type.** The name does not match Reference Data. Spelling and capitalization must match. For column rules and date formats, see [CSV file guidelines](/docs/guides/troubleshooting/csv-guidelines/). diff --git a/packages/web/src/lib/micronix-plate-positions.ts b/packages/web/src/lib/micronix-plate-positions.ts index 818b9e9d..feb9ca53 100644 --- a/packages/web/src/lib/micronix-plate-positions.ts +++ b/packages/web/src/lib/micronix-plate-positions.ts @@ -1,19 +1,16 @@ /** - * Well position utilities for micronix plates (96-well: A01–H12). - * Structured so other layouts (e.g. 384-well) can be added later. + * Well position utilities for micronix plates (96-well: A01–H12), on the shared Grid position + * rule from `@sampledb/contract`. */ +import { GRID_GEOMETRY, parseGridPosition } from '@sampledb/contract' -const ROWS_96 = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] as const -const COLS_96 = 12 +const { rows: PLATE_ROWS, columns: PLATE_COLUMNS } = GRID_GEOMETRY.micronix_plate /** Set of all 96 well positions for a standard micronix plate (8×12). */ export const MICRONIX_WELL_POSITIONS_96: Set = new Set( - ROWS_96.flatMap((row) => - Array.from({ length: COLS_96 }, (_, i) => { - const col = i + 1 - return `${row}${col.toString().padStart(2, '0')}` - }) - ) + Array.from({ length: PLATE_ROWS }, (_, r) => + Array.from({ length: PLATE_COLUMNS }, (_, c) => `${String.fromCharCode(65 + r)}${String(c + 1).padStart(2, '0')}`), + ).flat(), ) /** @@ -21,14 +18,8 @@ export const MICRONIX_WELL_POSITIONS_96: Set = new Set( * Returns normalized position or null if invalid. */ export function normalizeWellPosition(pos: string): string | null { - const t = pos.trim() - if (!t) return null - const match = t.match(/^([A-Ha-h])(\d{1,2})$/i) - if (!match) return null - const row = match[1].toUpperCase() - const col = parseInt(match[2], 10) - if (Number.isNaN(col) || col < 1 || col > 12) return null - return `${row}${col.toString().padStart(2, '0')}` + const parsed = parseGridPosition(pos, 'micronix_plate') + return parsed.ok ? parsed.position : null } export interface FullPlateValidationResult { diff --git a/packages/web/src/lib/scan-move/__tests__/ingest.test.ts b/packages/web/src/lib/scan-move/__tests__/ingest.test.ts index 3dec059a..3b938cae 100644 --- a/packages/web/src/lib/scan-move/__tests__/ingest.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/ingest.test.ts @@ -82,6 +82,20 @@ describe('cryovial ingest', () => { expect(file.inferredMatches).toEqual([]) }) + it('flags a target position outside the box or not a grid position, by CSV row', () => { + const text = [ + 'source_collection_name,source_position,target_position', + 'BOX-BETA,--A01,j10', + 'BOX-BETA,A02,K01', + 'BOX-BETA,A03,hello world', + ].join('\n') + const file = ingestScanCsvText(cryovialScanMoveVariant, fileSource('box-alpha.csv'), text, { collections: boxes }) + expect(file.validationErrors).toEqual([ + { row: 2, error: 'Position "K01" is outside a cryovial box (A01–J10)' }, + { row: 3, error: expect.stringContaining('Position "hello world" is not a grid position') }, + ]) + }) + it('requires a manual choice when multiple boxes match the filename case-insensitively', () => { const text = 'source_collection_name,source_position,target_position\nBOX-BETA,A01,B01' const file = ingestScanCsvText(cryovialScanMoveVariant, fileSource('box-alpha.csv'), text, { diff --git a/packages/web/src/lib/scan-move/variants.ts b/packages/web/src/lib/scan-move/variants.ts index a0b98276..2da9dfa6 100644 --- a/packages/web/src/lib/scan-move/variants.ts +++ b/packages/web/src/lib/scan-move/variants.ts @@ -1,3 +1,4 @@ +import { parseGridPosition } from '@sampledb/contract' import { inferDestinationPlateForScan } from '../plate-destination-inference' import { parseScannerPlateCsv, validateScannerPlateCsv } from '../scanner-plate-csv' import { parseBuiltinMoveCsv } from './ingest' @@ -89,7 +90,15 @@ export const cryovialScanMoveVariant: ScanMoveVariant = { capabilities: { createDestinations: true, relocationValidation: false }, parseAndValidate(text) { - return parseBuiltinMoveCsv(text, CRYOVIAL_MOVE_CSV_SPEC) + const { csvRows, errors } = parseBuiltinMoveCsv(text, CRYOVIAL_MOVE_CSV_SPEC) + // Grid position rule on the target only: a legacy source position must still resolve. + csvRows.forEach((row, i) => { + const target = csvCell(row, 'target_position') + if (target === '') return + const parsed = parseGridPosition(target, 'cryovial_box') + if (!parsed.ok) errors.push({ row: i + 1, error: parsed.error }) + }) + return { csvRows, errors } }, inferDestination(filename, _csvRows, ctx): ScanMoveInference { From 16ea9821b135769c0b14c0e0f728ffd3e72e1ada Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:33:43 -0700 Subject: [PATCH 06/54] fix(collections): show unplaced Containers on plate and box pages (#103) The micronix plate detail keyed wells by `position || ''`, so tubes without a Grid position overwrote each other and plate 103 showed all 96 tubes as Empty. The cryovial box detail dropped tubes with non-grid positions from the grid and drew only rows that held a tube. - @sampledb/contract: add collection-grid (plate A-H x 1-12, cryovial box A-J x 1-10) and gridPositionKey, which maps a stored position to its cell or null. - API: plate and box detail return placed Containers keyed by cell plus an `unplaced` list (missing or non-grid position, labelled with the stored value). Plate scan validation counts unplaced tubes as expected: a scanned one gets status `unplaced_in_scan`, not `extra_in_scan`, and the result lists every unplaced tube. - Web: plate and box pages render an Unplaced section, and the table view and its CSV export append unplaced rows. The cryovial grid always draws the full 10x10 box. Scan validation shows the new status and an Unplaced tubes table. - CONTEXT.md gains an "Unplaced container" entry; user guide updated. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 4 + .../__tests__/plate-scan-validation.test.ts | 119 ++++++++++++++++ .../collection-detail-unplaced.test.ts | 133 ++++++++++++++++++ .../src/lib/collections/collection-detail.ts | 37 ++++- packages/api/src/lib/plate-scan-validation.ts | 99 +++++++++---- .../src/routes/__tests__/collections.test.ts | 85 +++++++++++ packages/api/src/types/collections.ts | 6 + .../src/__tests__/collection-grid.test.ts | 54 +++++++ packages/contract/src/collection-grid.ts | 53 +++++++ packages/contract/src/index.ts | 8 ++ .../guides/features/plate-scan-validation.md | 7 +- .../docs/guides/workflows/containers.md | 2 +- .../components/UnplacedContainersSection.tsx | 83 +++++++++++ packages/web/src/lib/api/collections.ts | 31 +++- packages/web/src/pages/CryovialBoxDetail.tsx | 54 ++++--- .../web/src/pages/MicronixPlateDetail.tsx | 30 +++- .../web/src/pages/PlateScanValidation.tsx | 54 ++++++- .../__tests__/CryovialBoxDetail.test.tsx | 80 ++++++++++- .../__tests__/MicronixPlateDetail.test.tsx | 78 +++++++++- .../__tests__/PlateScanValidation.test.tsx | 56 +++++++- 20 files changed, 994 insertions(+), 79 deletions(-) create mode 100644 packages/api/src/lib/collections/__tests__/collection-detail-unplaced.test.ts create mode 100644 packages/contract/src/__tests__/collection-grid.test.ts create mode 100644 packages/contract/src/collection-grid.ts create mode 100644 packages/web/src/components/UnplacedContainersSection.tsx diff --git a/CONTEXT.md b/CONTEXT.md index 99674c4f..50d59dba 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -135,6 +135,10 @@ _Avoid_: Enrichment (implementation term); conflating placement with specimen or The well or slot of a tube or well **Container** in its plate or box (e.g. A01). Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. _Avoid_: Well (as the domain term); treating a missing position as occupying a slot; requiring every tube to have a position before it can exist in a Collection +**Unplaced container**: +A tube or well **Container** that belongs to a plate or box but occupies no cell: it has no **Grid position**, or its stored position is not a cell of the Collection's grid (rows A–H, columns 1–12 on a micronix plate; rows A–J, columns 1–10 on a cryovial box). It still counts as part of the Collection. Plate and box pages list unplaced Containers in their own section, labelled with the stored value, and plate scan validation expects them rather than treating them as extras. +_Avoid_: Empty (an empty cell holds no Container); treating an unplaced Container as missing from its Collection + ### Export **Container selection**: diff --git a/packages/api/src/lib/__tests__/plate-scan-validation.test.ts b/packages/api/src/lib/__tests__/plate-scan-validation.test.ts index f509f2fb..db3a2f37 100644 --- a/packages/api/src/lib/__tests__/plate-scan-validation.test.ts +++ b/packages/api/src/lib/__tests__/plate-scan-validation.test.ts @@ -731,4 +731,123 @@ describe('inferPlateOrGetReport', () => { expect(result).not.toHaveProperty('inferenceReport') expect((result as { plate: { id: number; name: string } }).plate).toEqual({ id: plate.id, name: 'Plate1' }) }) + + it('counts unplaced tubes of the plate as expected, not as extras', async () => { + await setScannerConfigurations(testDb, { + configurations: [ + { + id: 'test-config', + name: 'Test', + barcodeColumn: 'Barcode', + positionType: 'single', + positionColumn: 'Well', + skipRows: 0, + }, + ], + }, null) + + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { + name: 'Loc', + storageTypeId: String(storageType.id), + }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: location.id }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const specimen = await createTestSpecimen(testDb, specimenType.id) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const tubes: Array<[string, string | null, number]> = [ + ['MT-PLACED', 'A01', 1], + ['MT-NULL-1', null, 1], + ['MT-NULL-2', null, 0], + ['MT-BAD-POS', 'A1145868849', 1], + ] + for (const [barcode, position, remainingQuantity] of tubes) { + const now = utcNow() + const [container] = await testDb + .insert(storageContainer) + .values({ + specimenId: specimen.id, + unitId: unit.id, + totalQuantity: 1.0, + remainingQuantity, + created: now, + lastUpdated: now, + }) + .returning() + await testDb.insert(micronixTube).values({ id: container!.id, collectionId: plate.id, barcode, position }) + } + + const result = await validatePlateScan(testDb, { + csvText: 'Well,Barcode\nA01,MT-PLACED\nB01,MT-NULL-1\nC01,MT-NULL-2\nD01,STRANGER', + plateId: plate.id, + scannerConfigurationId: 'test-config', + }) + + expect(result.summary).toMatchObject({ + totalExpected: 4, + matched: 1, + unplacedInScan: 2, + extraInScan: 1, + missingInScan: 1, + mismatch: 0, + exhaustedCount: 1, + }) + expect(result.wells.map((w) => [w.position, w.status])).toEqual([ + ['A01', 'match'], + ['B01', 'unplaced_in_scan'], + ['C01', 'unplaced_in_scan'], + ['D01', 'extra_in_scan'], + ]) + const c01 = result.wells.find((w) => w.position === 'C01')! + expect(c01.exhausted).toBe(true) + expect(c01.scanBarcodeOrigin).toEqual({ plateId: plate.id, plateName: 'Plate1', position: '' }) + expect(result.unplaced).toEqual([ + { barcode: 'MT-NULL-1', storedPosition: null, scannedPosition: 'B01' }, + { barcode: 'MT-NULL-2', storedPosition: null, scannedPosition: 'C01' }, + { barcode: 'MT-BAD-POS', storedPosition: 'A1145868849', scannedPosition: null }, + ]) + }) + + it('still reports a mismatch when an unplaced tube sits where a placed tube is expected', async () => { + await setScannerConfigurations(testDb, { + configurations: [ + { + id: 'test-config', + name: 'Test', + barcodeColumn: 'Barcode', + positionType: 'single', + positionColumn: 'Well', + skipRows: 0, + }, + ], + }, null) + + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { + name: 'Loc', + storageTypeId: String(storageType.id), + }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: location.id }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const specimen = await createTestSpecimen(testDb, specimenType.id) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + for (const [barcode, position] of [['MT-A01', 'A01'], ['MT-NULL', null]] as const) { + const now = utcNow() + const [container] = await testDb + .insert(storageContainer) + .values({ specimenId: specimen.id, unitId: unit.id, totalQuantity: 1, remainingQuantity: 1, created: now, lastUpdated: now }) + .returning() + await testDb.insert(micronixTube).values({ id: container!.id, collectionId: plate.id, barcode, position }) + } + + const result = await validatePlateScan(testDb, { + csvText: 'Well,Barcode\nA01,MT-NULL', + plateId: plate.id, + scannerConfigurationId: 'test-config', + }) + + expect(result.wells.map((w) => [w.position, w.status])).toEqual([['A01', 'mismatch']]) + expect(result.summary).toMatchObject({ totalExpected: 2, mismatch: 1, unplacedInScan: 1, extraInScan: 0, missingInScan: 0 }) + expect(result.unplaced).toEqual([{ barcode: 'MT-NULL', storedPosition: null, scannedPosition: 'A01' }]) + }) }) diff --git a/packages/api/src/lib/collections/__tests__/collection-detail-unplaced.test.ts b/packages/api/src/lib/collections/__tests__/collection-detail-unplaced.test.ts new file mode 100644 index 00000000..aa87617c --- /dev/null +++ b/packages/api/src/lib/collections/__tests__/collection-detail-unplaced.test.ts @@ -0,0 +1,133 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' +import { + createTestCryovialBox, + createTestCryovialTube, + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestSpecimen, + createTestSpecimenType, + createTestStorageContainer, + createTestStorageType, + createTestUnit, +} from '../../../__tests__/helpers/factories' +import type { Database } from '../../../db/client' +import { getCryovialBoxDetail, getMicronixPlateDetail } from '../collection-detail' + +describe('collection detail with unplaced Containers', () => { + let testDb: Database + let sqlite: Awaited>['sqlite'] + let locationId: number + let containerOptions: { specimenId: number; unitId: number } + + beforeEach(async () => { + const setup = await setupTestDatabase() + testDb = setup.db + sqlite = setup.sqlite + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const loc = await createTestLocation(testDb, { + name: 'Freezer', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + locationId = loc.id + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const specimen = await createTestSpecimen(testDb, specimenType.id) + containerOptions = { specimenId: specimen.id, unitId: unit.id } + }) + + afterEach(() => { + cleanupTestDatabase(sqlite) + }) + + async function addPlateTube(plateId: number, barcode: string, position: string | null) { + const container = await createTestStorageContainer(testDb, containerOptions) + await createTestMicronixTube(testDb, { containerId: container.id, collectionId: plateId, barcode, position }) + return container.id + } + + async function addBoxTube(boxId: number, barcode: string, position: string | null) { + const container = await createTestStorageContainer(testDb, containerOptions) + await createTestCryovialTube(testDb, { containerId: container.id, collectionId: boxId, barcode, position }) + return container.id + } + + it('lists plate tubes without a grid position separately from placed tubes', async () => { + const plate = await createTestMicronixPlate(testDb, { name: 'P-unplaced', locationId }) + const placedId = await addPlateTube(plate.id, 'MX-PLACED', 'B07') + await addPlateTube(plate.id, 'MX-NULL-2', null) + await addPlateTube(plate.id, 'MX-NULL-1', null) + await addPlateTube(plate.id, 'MX-NULL-3', null) + + const detail = await getMicronixPlateDetail(testDb, plate.id) + + expect(Object.keys(detail!.wells)).toEqual(['B07']) + expect(detail!.wells.B07!.id).toBe(placedId) + expect(detail!.unplaced.map((e) => e.barcode)).toEqual(['MX-NULL-1', 'MX-NULL-2', 'MX-NULL-3']) + expect(detail!.unplaced.every((e) => e.position === null)).toBe(true) + expect(detail!.unplaced.every((e) => e.container != null)).toBe(true) + }) + + it('keeps stored plate positions that are not grid cells as unplaced with their stored value', async () => { + const plate = await createTestMicronixPlate(testDb, { name: 'P-invalid', locationId }) + await addPlateTube(plate.id, 'MX-A1', 'a1') + await addPlateTube(plate.id, 'MX-BARCODE-POS', 'A1145868849') + await addPlateTube(plate.id, 'MX-ROW-I', 'I01') + await addPlateTube(plate.id, 'MX-COL-13', 'A13') + + const detail = await getMicronixPlateDetail(testDb, plate.id) + + expect(Object.keys(detail!.wells)).toEqual(['A01']) + expect(detail!.wells.A01!.barcode).toBe('MX-A1') + expect(detail!.unplaced.map((e) => [e.position, e.barcode])).toEqual([ + ['A13', 'MX-COL-13'], + ['A1145868849', 'MX-BARCODE-POS'], + ['I01', 'MX-ROW-I'], + ]) + }) + + it('lists a second plate tube normalizing to an occupied cell as unplaced instead of dropping it', async () => { + const plate = await createTestMicronixPlate(testDb, { name: 'P-dup', locationId }) + await addPlateTube(plate.id, 'MX-FIRST', 'C03') + await addPlateTube(plate.id, 'MX-SECOND', 'c3') + + const detail = await getMicronixPlateDetail(testDb, plate.id) + + expect(detail!.wells.C03!.barcode).toBe('MX-FIRST') + expect(detail!.unplaced.map((e) => e.barcode)).toEqual(['MX-SECOND']) + }) + + it('returns an empty unplaced list when every plate tube has a grid cell', async () => { + const plate = await createTestMicronixPlate(testDb, { name: 'P-full', locationId }) + await addPlateTube(plate.id, 'MX-1', 'A01') + + const detail = await getMicronixPlateDetail(testDb, plate.id) + + expect(detail!.unplaced).toEqual([]) + }) + + it('lists cryovial tubes with missing or non-grid positions as unplaced, keeping the stored value', async () => { + const box = await createTestCryovialBox(testDb, { name: 'B-unplaced', locationId }) + await addBoxTube(box.id, 'CV-B7', 'B7') + await addBoxTube(box.id, 'CV-J10', 'J10') + await addBoxTube(box.id, 'CV-NULL', null) + await addBoxTube(box.id, 'CV-DASH', '--A01') + await addBoxTube(box.id, 'CV-NA', 'NA01') + await addBoxTube(box.id, 'CV-K', 'K01') + await addBoxTube(box.id, 'CV-COL11', 'A11') + + const detail = await getCryovialBoxDetail(testDb, box.id) + + expect(Object.keys(detail!.positions).sort()).toEqual(['B07', 'J10']) + expect(detail!.positions.B07!.map((e) => e.barcode)).toEqual(['CV-B7']) + expect(detail!.unplaced.map((e) => [e.position, e.barcode])).toEqual([ + ['--A01', 'CV-DASH'], + ['A11', 'CV-COL11'], + ['K01', 'CV-K'], + ['NA01', 'CV-NA'], + [null, 'CV-NULL'], + ]) + }) +}) diff --git a/packages/api/src/lib/collections/collection-detail.ts b/packages/api/src/lib/collections/collection-detail.ts index b4d8b0c6..38cde873 100644 --- a/packages/api/src/lib/collections/collection-detail.ts +++ b/packages/api/src/lib/collections/collection-detail.ts @@ -12,10 +12,26 @@ import { sheet, location, } from '../../db/schema' +import { CRYOVIAL_BOX_GRID, MICRONIX_PLATE_GRID, gridPositionKey } from '@sampledb/contract' import { formatLocationPath } from '../container-enrichment' -import { normalizePosition } from '../normalize-position' import { enrichPaperContainers, enrichStorageContainers } from './container-detail' +const naturalOrder = new Intl.Collator(undefined, { numeric: true, sensitivity: 'base' }) + +/** Unplaced entries by stored position (missing last), then barcode, then id. */ +function sortUnplaced(entries: T[]): T[] { + return [...entries].sort((a, b) => { + if (a.position !== b.position) { + if (a.position == null) return 1 + if (b.position == null) return -1 + const byPosition = naturalOrder.compare(a.position, b.position) + if (byPosition !== 0) return byPosition + } + const byBarcode = naturalOrder.compare(a.barcode ?? '', b.barcode ?? '') + return byBarcode !== 0 ? byBarcode : a.id - b.id + }) +} + export async function getMicronixPlateDetail(database: Database, id: number) { const plate = await database.select().from(micronixPlate).where(eq(micronixPlate.id, id)).get() if (!plate) return null @@ -46,9 +62,16 @@ export async function getMicronixPlateDetail(database: Database, id: number) { container: enrichedById.get(w.id) ?? null, })) - const wellsByPosition: Record = {} + // A Container whose position is missing or not a cell of the plate grid is + // unplaced. A second Container normalizing to an occupied cell is also listed + // as unplaced so that no Container drops out of the response. + type PlateEntry = (typeof tubeEntries)[number] | (typeof wellEntries)[number] + const wellsByPosition: Record = {} + const unplaced: PlateEntry[] = [] for (const entry of [...tubeEntries, ...wellEntries]) { - wellsByPosition[entry.position || ''] = entry + const key = gridPositionKey(entry.position, MICRONIX_PLATE_GRID) + if (key != null && !(key in wellsByPosition)) wellsByPosition[key] = entry + else unplaced.push(entry) } return { @@ -58,6 +81,7 @@ export async function getMicronixPlateDetail(database: Database, id: number) { locationPath: formatLocationPath(loc), }, wells: wellsByPosition, + unplaced: sortUnplaced(unplaced), } } @@ -78,9 +102,11 @@ export async function getCryovialBoxDetail(database: Database, id: number) { })) const positions: Record = {} + const unplaced: typeof tubeEntries = [] for (const entry of tubeEntries) { - const pos = normalizePosition(entry.position) ?? '' - ;(positions[pos] ??= []).push(entry) + const key = gridPositionKey(entry.position, CRYOVIAL_BOX_GRID) + if (key == null) unplaced.push(entry) + else (positions[key] ??= []).push(entry) } return { @@ -90,6 +116,7 @@ export async function getCryovialBoxDetail(database: Database, id: number) { locationPath: formatLocationPath(loc), }, positions, + unplaced: sortUnplaced(unplaced), } } diff --git a/packages/api/src/lib/plate-scan-validation.ts b/packages/api/src/lib/plate-scan-validation.ts index 25143795..19d257b1 100644 --- a/packages/api/src/lib/plate-scan-validation.ts +++ b/packages/api/src/lib/plate-scan-validation.ts @@ -8,10 +8,15 @@ import { tag, } from '../db/schema' import { eq, inArray } from 'drizzle-orm' +import { MICRONIX_PLATE_GRID, gridCellKeys, gridPositionKey } from '@sampledb/contract' import { getScannerConfigurationById } from './settings' import { parsePlateCSV, normalizeWellPosition } from './plate-csv' -export type WellValidationStatus = 'match' | 'mismatch' | 'missing_in_scan' | 'extra_in_scan' +/** + * `unplaced_in_scan`: the scanned tube belongs to this plate but has no grid position + * (or a stored position that is not a plate cell), so it is expected, not extra. + */ +export type WellValidationStatus = 'match' | 'mismatch' | 'missing_in_scan' | 'extra_in_scan' | 'unplaced_in_scan' /** Where the scanned barcode is registered (plate + position), when it differs from expected or is extra. */ export interface ScanBarcodeOrigin { @@ -32,19 +37,33 @@ export interface WellValidationResult { } export interface ValidateScanSummary { + /** Every tube in the plate, placed or unplaced, plus placed static wells. */ totalExpected: number matched: number + /** Placed wells empty in the scan, plus unplaced tubes whose barcode is nowhere in the scan. */ missingInScan: number extraInScan: number + /** Unplaced tubes of this plate whose barcode the scan found at some position. */ + unplacedInScan: number mismatch: number exhaustedCount: number taggedCount: number } +/** A tube in the plate with no grid position, or a stored position that is not a plate cell. */ +export interface UnplacedTubeResult { + barcode: string + /** The stored position as recorded, or null when none is recorded. */ + storedPosition: string | null + /** Where the scan found this barcode, or null when it is not in the scan. */ + scannedPosition: string | null +} + export interface ValidateScanResult { plate: { id: number; name: string } summary: ValidateScanSummary wells: WellValidationResult[] + unplaced: UnplacedTubeResult[] } export interface InferPlateResult { @@ -254,15 +273,29 @@ export async function validatePlateScan( ...wells.map((w) => w.id), ] - const expectedByPosition = new Map() + const expectedByPosition = new Map() + // Tubes whose position is missing or not a plate cell (or a cell another tube already holds). + const unplacedByBarcode = new Map & { barcode: string; storedPosition: string | null; containerId: number }>() for (const t of tubes) { - const pos = t.position ? normalizeWellPosition(t.position) : null - if (pos) expectedByPosition.set(pos, { position: pos, barcode: t.barcode, remainingQuantity: null, tags: [] }) + const pos = gridPositionKey(t.position, MICRONIX_PLATE_GRID) + if (pos != null && !expectedByPosition.has(pos)) { + expectedByPosition.set(pos, { position: pos, containerId: t.id, barcode: t.barcode, remainingQuantity: null, tags: [] }) + } else { + unplacedByBarcode.set(t.barcode, { + storedPosition: t.position, + containerId: t.id, + barcode: t.barcode, + remainingQuantity: null, + tags: [], + }) + } } for (const w of wells) { - const pos = w.position ? normalizeWellPosition(w.position) : null - if (pos) expectedByPosition.set(pos, { position: pos, barcode: null, remainingQuantity: null, tags: [] }) + const pos = gridPositionKey(w.position, MICRONIX_PLATE_GRID) + if (pos != null && !expectedByPosition.has(pos)) { + expectedByPosition.set(pos, { position: pos, containerId: w.id, barcode: null, remainingQuantity: null, tags: [] }) + } } if (containerIds.length > 0) { @@ -283,19 +316,17 @@ export async function validatePlateScan( tagsByContainerId.set(tr.storageContainerId, list) } - for (const t of tubes) { - const entry = expectedByPosition.get(normalizeWellPosition(t.position ?? '')) - if (entry) { - entry.remainingQuantity = remainingByContainerId.get(t.id) ?? null - entry.tags = tagsByContainerId.get(t.id) ?? [] - } + for (const entry of [...expectedByPosition.values(), ...unplacedByBarcode.values()]) { + entry.remainingQuantity = remainingByContainerId.get(entry.containerId) ?? null + entry.tags = tagsByContainerId.get(entry.containerId) ?? [] } - for (const w of wells) { - const entry = expectedByPosition.get(normalizeWellPosition(w.position ?? '')) - if (entry) { - entry.remainingQuantity = remainingByContainerId.get(w.id) ?? null - entry.tags = tagsByContainerId.get(w.id) ?? [] - } + } + + const scannedPositionByUnplacedBarcode = new Map() + for (const [pos, barcode] of scannedByPosition) { + const trimmed = barcode.trim() + if (unplacedByBarcode.has(trimmed) && !scannedPositionByUnplacedBarcode.has(trimmed)) { + scannedPositionByUnplacedBarcode.set(trimmed, pos) } } @@ -305,16 +336,14 @@ export async function validatePlateScan( if (barcode.trim() !== '') allPositions.add(pos) else if (expectedByPosition.has(pos)) allPositions.add(pos) } - const ROWS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] - const sortedPositions = ROWS.flatMap((r) => - Array.from({ length: 12 }, (_, i) => `${r}${(i + 1).toString().padStart(2, '0')}`) - ).filter((p) => allPositions.has(p)) + const sortedPositions = gridCellKeys(MICRONIX_PLATE_GRID).filter((p) => allPositions.has(p)) const rest = [...allPositions].filter((p) => !sortedPositions.includes(p)).sort() const positionOrder = [...sortedPositions, ...rest] let matched = 0 let missingInScan = 0 let extraInScan = 0 + let unplacedInScan = 0 let mismatch = 0 let exhaustedCount = 0 let taggedCount = 0 @@ -328,8 +357,10 @@ export async function validatePlateScan( const scanHasBarcode = scanPresent && (scanBarcodeRaw ?? '').trim() !== '' const scanBarcode = scanPresent ? (scanBarcodeRaw ?? null) : null const expectedBarcode = expected?.barcode ?? null - const exhausted = expected != null && expected.remainingQuantity != null && expected.remainingQuantity <= 0 - const tagsList = expected?.tags ?? [] + const unplacedTube = !expected && scanHasBarcode ? unplacedByBarcode.get((scanBarcode ?? '').trim()) : undefined + const described = expected ?? unplacedTube + const exhausted = described != null && described.remainingQuantity != null && described.remainingQuantity <= 0 + const tagsList = described?.tags ?? [] if (exhausted) exhaustedCount++ if (tagsList.length > 0) taggedCount++ @@ -345,6 +376,8 @@ export async function validatePlateScan( } else if (expected && (!scanPresent || !scanHasBarcode)) { status = 'missing_in_scan' missingInScan++ + } else if (unplacedTube) { + status = 'unplaced_in_scan' } else { status = 'extra_in_scan' extraInScan++ @@ -367,7 +400,7 @@ export async function validatePlateScan( wellResults .filter( (w) => - (w.status === 'mismatch' || w.status === 'extra_in_scan') && + (w.status === 'mismatch' || w.status === 'extra_in_scan' || w.status === 'unplaced_in_scan') && w.scanBarcode != null && w.scanBarcode.trim() !== '' ) @@ -400,7 +433,7 @@ export async function validatePlateScan( } for (const w of wellResults) { if ( - (w.status === 'mismatch' || w.status === 'extra_in_scan') && + (w.status === 'mismatch' || w.status === 'extra_in_scan' || w.status === 'unplaced_in_scan') && w.scanBarcode != null && w.scanBarcode.trim() !== '' ) { @@ -409,12 +442,23 @@ export async function validatePlateScan( } } - const totalExpected = expectedByPosition.size + const unplaced: UnplacedTubeResult[] = [...unplacedByBarcode.values()].map((t) => ({ + barcode: t.barcode, + storedPosition: t.storedPosition, + scannedPosition: scannedPositionByUnplacedBarcode.get(t.barcode) ?? null, + })) + for (const t of unplaced) { + if (t.scannedPosition != null) unplacedInScan++ + else missingInScan++ + } + + const totalExpected = expectedByPosition.size + unplacedByBarcode.size const summary: ValidateScanSummary = { totalExpected, matched, missingInScan, extraInScan, + unplacedInScan, mismatch, exhaustedCount, taggedCount, @@ -424,5 +468,6 @@ export async function validatePlateScan( plate: { id: plate.id, name: plate.name }, summary, wells: wellResults, + unplaced, } } diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 7a80c107..694771fb 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -17,6 +17,10 @@ import { createTestStudySubject, createTestBox, createTestSheet, + createTestStorageContainer, + createTestMicronixTube, + createTestCryovialBox, + createTestCryovialTube, } from '../../__tests__/helpers/factories' import { setScannerConfigurations } from '../../lib/settings' import { eq } from 'drizzle-orm' @@ -110,6 +114,87 @@ describe('Collections API', () => { }) }) + describe('GET collection detail with unplaced Containers', () => { + async function setupCollectionFixtures() { + const storageType = await createTestStorageType(ctx.db, { name: 'Unplaced freezer' }) + const loc = await createTestLocation(ctx.db, { + name: 'Unplaced freezer', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const testSpecimen = await createTestSpecimen(ctx.db, specimenType.id) + const newContainer = () => + createTestStorageContainer(ctx.db, { specimenId: testSpecimen.id, unitId: unit.id }) + return { locationId: loc.id, newContainer } + } + + it('returns every plate tube, listing those without a grid cell under unplaced', async () => { + const { locationId, newContainer } = await setupCollectionFixtures() + const plate = await createTestMicronixPlate(ctx.db, { name: 'Plate-unplaced', locationId }) + const tubes: Array<[string, string | null]> = [ + ['MX-A01', 'A01'], + ['MX-NULL-1', null], + ['MX-NULL-2', null], + ['MX-BAD', 'A1145868849'], + ] + for (const [barcode, position] of tubes) { + const c = await newContainer() + await createTestMicronixTube(ctx.db, { containerId: c.id, collectionId: plate.id, barcode, position }) + } + + const res = await ctx.request(`/api/collections/plates/micronix/${plate.id}`, { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { + wells: Record + unplaced: Array<{ barcode: string; position?: string; container?: unknown }> + } + expect(Object.keys(data.wells)).toEqual(['A01']) + expect(data.unplaced.map((e) => [e.barcode, e.position ?? null])).toEqual([ + ['MX-BAD', 'A1145868849'], + ['MX-NULL-1', null], + ['MX-NULL-2', null], + ]) + expect(data.unplaced.every((e) => e.container != null)).toBe(true) + }) + + it('returns every cryovial tube, listing those without a grid cell under unplaced', async () => { + const { locationId, newContainer } = await setupCollectionFixtures() + const boxRecord = await createTestCryovialBox(ctx.db, { name: 'Box-unplaced', locationId }) + const tubes: Array<[string, string | null]> = [ + ['CV-J10', 'J10'], + ['CV-NULL', null], + ['CV-DASH', '--A01'], + ['CV-NA', 'NA01'], + ] + for (const [barcode, position] of tubes) { + const c = await newContainer() + await createTestCryovialTube(ctx.db, { containerId: c.id, collectionId: boxRecord.id, barcode, position }) + } + + const res = await ctx.request(`/api/collections/boxes/cryovial/${boxRecord.id}`, { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { + positions: Record> + unplaced: Array<{ barcode: string; position?: string }> + } + expect(Object.keys(data.positions)).toEqual(['J10']) + expect(data.unplaced.map((e) => [e.barcode, e.position ?? null])).toEqual([ + ['CV-DASH', '--A01'], + ['CV-NA', 'NA01'], + ['CV-NULL', null], + ]) + }) + + it('returns 404 for a missing cryovial box', async () => { + const res = await ctx.request('/api/collections/boxes/cryovial/99999', { method: 'GET' }) + expect(res.status).toBe(404) + }) + }) + describe('POST /api/collections/check', () => { it('returns 200 and results for check request', async () => { const res = await ctx.request('/api/collections/check', { diff --git a/packages/api/src/types/collections.ts b/packages/api/src/types/collections.ts index 93e90e1d..cefb4bb4 100644 --- a/packages/api/src/types/collections.ts +++ b/packages/api/src/types/collections.ts @@ -141,7 +141,10 @@ export interface SheetResponse extends Sheet { */ export interface MicronixPlateDetailResponse { plate: MicronixPlateResponse + /** Placed Containers keyed by grid cell (A01–H12). */ wells: Record + /** Containers with no grid position, or a stored position that is not a plate cell. */ + unplaced: WellEntry[] } /** @@ -149,7 +152,10 @@ export interface MicronixPlateDetailResponse { */ export interface CryovialBoxDetailResponse { box: CryovialBoxResponse + /** Placed Containers keyed by grid cell (A01–J10). */ positions: Record + /** Containers with no grid position, or a stored position that is not a box cell. */ + unplaced: CryovialTubeEntry[] } /** diff --git a/packages/contract/src/__tests__/collection-grid.test.ts b/packages/contract/src/__tests__/collection-grid.test.ts new file mode 100644 index 00000000..42e554f7 --- /dev/null +++ b/packages/contract/src/__tests__/collection-grid.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'bun:test' +import { + CRYOVIAL_BOX_GRID, + MICRONIX_PLATE_GRID, + gridCellKeys, + gridPositionKey, +} from '../collection-grid' + +describe('collection grid geometry', () => { + it('orders cryovial box rows and columns naturally', () => { + expect(CRYOVIAL_BOX_GRID.rows).toEqual(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']) + expect(CRYOVIAL_BOX_GRID.columns).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) + }) + + it('lists every cell of a micronix plate row by row', () => { + const keys = gridCellKeys(MICRONIX_PLATE_GRID) + expect(keys).toHaveLength(96) + expect(keys.slice(0, 3)).toEqual(['A01', 'A02', 'A03']) + expect(keys[12]).toBe('B01') + expect(keys.at(-1)).toBe('H12') + }) +}) + +describe('gridPositionKey', () => { + it('normalizes a valid position to its cell key', () => { + expect(gridPositionKey('A01', MICRONIX_PLATE_GRID)).toBe('A01') + expect(gridPositionKey('a1', MICRONIX_PLATE_GRID)).toBe('A01') + expect(gridPositionKey(' H12 ', MICRONIX_PLATE_GRID)).toBe('H12') + expect(gridPositionKey('J10', CRYOVIAL_BOX_GRID)).toBe('J10') + }) + + it('returns null for a missing position', () => { + expect(gridPositionKey(null, MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey(undefined, MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey('', MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey(' ', MICRONIX_PLATE_GRID)).toBeNull() + }) + + it('returns null for a position outside the grid', () => { + expect(gridPositionKey('I01', MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey('A13', MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey('A00', MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey('K01', CRYOVIAL_BOX_GRID)).toBeNull() + expect(gridPositionKey('A11', CRYOVIAL_BOX_GRID)).toBeNull() + expect(gridPositionKey('Z99', CRYOVIAL_BOX_GRID)).toBeNull() + }) + + it('returns null for stored values that are not a row letter plus column number', () => { + expect(gridPositionKey('--A01', CRYOVIAL_BOX_GRID)).toBeNull() + expect(gridPositionKey('NA01', CRYOVIAL_BOX_GRID)).toBeNull() + expect(gridPositionKey('A1145868849', MICRONIX_PLATE_GRID)).toBeNull() + expect(gridPositionKey('hello world', MICRONIX_PLATE_GRID)).toBeNull() + }) +}) diff --git a/packages/contract/src/collection-grid.ts b/packages/contract/src/collection-grid.ts new file mode 100644 index 00000000..e2da2b4d --- /dev/null +++ b/packages/contract/src/collection-grid.ts @@ -0,0 +1,53 @@ +/** + * Grid geometry of the Collections that hold tube and well Containers, and the + * rule that maps a stored grid position onto that geometry. + * + * A stored position that does not map onto the grid (missing, out of range, or + * not a row letter plus column number) leaves its Container unplaced: it + * belongs to the Collection but occupies no cell. + */ + +export type CollectionGrid = { + /** Row letters, top to bottom. */ + readonly rows: readonly string[] + /** Column numbers, left to right. */ + readonly columns: readonly number[] +} + +function letters(count: number): string[] { + return Array.from({ length: count }, (_, i) => String.fromCharCode(65 + i)) +} + +function numbers(count: number): number[] { + return Array.from({ length: count }, (_, i) => i + 1) +} + +/** 96-well micronix plate: rows A–H, columns 1–12. */ +export const MICRONIX_PLATE_GRID: CollectionGrid = { rows: letters(8), columns: numbers(12) } + +/** 100-slot cryovial box: rows A–J, columns 1–10. */ +export const CRYOVIAL_BOX_GRID: CollectionGrid = { rows: letters(10), columns: numbers(10) } + +/** Cell key for a row and column, e.g. `B` and `7` give `B07`. */ +export function gridCellKey(row: string, column: number): string { + return `${row}${String(column).padStart(2, '0')}` +} + +/** Every cell key of the grid, row by row (A01, A02, …, B01, …). */ +export function gridCellKeys(grid: CollectionGrid): string[] { + return grid.rows.flatMap((row) => grid.columns.map((column) => gridCellKey(row, column))) +} + +/** + * The grid cell a stored position names, or null when the Container is unplaced. + * `a1`, `A1` and ` A01 ` all give `A01`; `Z99`, `--A01`, `NA01` and null give null. + */ +export function gridPositionKey(position: string | null | undefined, grid: CollectionGrid): string | null { + if (position == null) return null + const match = /^([A-Za-z])(\d{1,3})$/.exec(position.trim()) + if (!match) return null + const row = match[1]!.toUpperCase() + const column = parseInt(match[2]!, 10) + if (!grid.rows.includes(row) || !grid.columns.includes(column)) return null + return gridCellKey(row, column) +} diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1a9f103e..bccb079b 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -81,3 +81,11 @@ export { PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type PaperInboundWriteFields, } from './paper-container-inbound' +export { + CRYOVIAL_BOX_GRID, + MICRONIX_PLATE_GRID, + gridCellKey, + gridCellKeys, + gridPositionKey, + type CollectionGrid, +} from './collection-grid' diff --git a/packages/docs/src/content/docs/guides/features/plate-scan-validation.md b/packages/docs/src/content/docs/guides/features/plate-scan-validation.md index f708cd67..871c750c 100644 --- a/packages/docs/src/content/docs/guides/features/plate-scan-validation.md +++ b/packages/docs/src/content/docs/guides/features/plate-scan-validation.md @@ -12,6 +12,7 @@ Use validate plate scan when you want to: - Check that a scanned plate CSV matches the plate record in the database. - Find wells that are missing from the scan (expected in DB but not scanned or scanned empty). - Find wells that appear in the scan but are not in the database (extra in scan). +- Find tubes that belong to the plate but have no grid position recorded (unplaced). - Spot barcode mismatches (same position, different barcode in scan vs DB). - See which wells are exhausted (remaining quantity zero) or have tags. @@ -37,16 +38,18 @@ Validation is read-only: it does not change any data. The result shows: -- **Summary counts**: Matched wells, mismatch (same position, different barcode), missing in scan (in DB but not in scan or scanned empty), and extra in scan (in scan but not in DB). If any wells are exhausted or have tags, those counts are shown as well. +- **Summary counts**: Matched wells, mismatch (same position, different barcode), missing in scan (in DB but not in scan or scanned empty), extra in scan (in scan but not in DB), and unplaced in plate (a scanned tube that belongs to this plate but has no grid position recorded). Unplaced tubes count as expected: one found in the scan is not an extra, and one missing from the scan counts as missing. If any wells are exhausted or have tags, those counts are shown as well. - **Overall status**: A short message indicates whether the scan matches the database or there are discrepancies. - **Well table**: Each row is a well (position). Columns show: - **Position** (e.g. A01) - **Scanned** barcode from the CSV (or — if empty) - **Expected** barcode from the database (or — for static wells) - **Scanned barcode from**: For mismatch or extra-in-scan, where that scanned barcode is registered in the database (plate name and position). Helps you see if a tube was misplaced from another plate. Shows — when the barcode is not in the database. - - **Status**: Match (green), Mismatch (red), Missing in scan (amber), Extra in scan (blue) + - **Status**: Match (green), Mismatch (red), Missing in scan (amber), Extra in scan (blue), Unplaced in plate (grey) - **Notes**: Badges for "Exhausted" and any tag names on the container +- **Unplaced tubes**: When the plate has tubes with no grid position, or a stored position that is not a plate cell, a second table lists each one with its stored position and where the scan found it (or **Not in scan**). The downloaded report includes the same list. + Use the table to track down specific wells that don’t match or are missing so you can correct the physical plate or the database as needed. ## When inference cannot determine a single plate diff --git a/packages/docs/src/content/docs/guides/workflows/containers.md b/packages/docs/src/content/docs/guides/workflows/containers.md index 33841342..d2314c33 100644 --- a/packages/docs/src/content/docs/guides/workflows/containers.md +++ b/packages/docs/src/content/docs/guides/workflows/containers.md @@ -64,7 +64,7 @@ Micronix barcodes must be unique across the system. Paper uses **sheet name** an Open **Browse Data → Collections**. Tabs: **All**, **Micronix Plates**, **Cryovial Boxes**, **Boxes**, **Bags**. See [Browsing collections](/docs/guides/features/browse-collections/). -Plate pages include a grid of occupied and empty wells. Box and bag pages list sheets. Every collection type has a **Table** view and **Export CSV**, which downloads the rows currently shown. +Micronix plate and cryovial box pages include a grid of occupied and empty cells. The grid always shows the whole plate (A–H, 1–12) or box (A–J, 1–10). Tubes that belong to the plate or box but have no position, or a stored position that is not a cell of the grid, appear below the grid in an **Unplaced** section, labelled with the stored value. Box and bag pages list sheets. Every collection type has a **Table** view and **Export CSV**, which downloads the rows currently shown; on plates and boxes this includes unplaced tubes after the grid rows. ## Delete a collection diff --git a/packages/web/src/components/UnplacedContainersSection.tsx b/packages/web/src/components/UnplacedContainersSection.tsx new file mode 100644 index 00000000..8a57eca8 --- /dev/null +++ b/packages/web/src/components/UnplacedContainersSection.tsx @@ -0,0 +1,83 @@ +import { useId } from 'react' +import { Link } from 'react-router-dom' +import type { CollectionEntryContainer } from '../lib/collection-table-columns' + +export interface UnplacedContainerEntry { + id: number + barcode?: string | null + /** The stored position as recorded, missing when none is recorded. */ + position?: string | null + container?: unknown +} + +export interface UnplacedContainersSectionProps { + entries: UnplacedContainerEntry[] + /** Collection kind for the explanatory text, e.g. "plate" or "box". */ + collectionLabel: string +} + +/** + * Containers that belong to a plate or box but occupy no grid cell: their grid + * position is missing, or the stored value is not a cell of the grid. + */ +export default function UnplacedContainersSection({ entries, collectionLabel }: UnplacedContainersSectionProps) { + const headingId = useId() + if (entries.length === 0) return null + + return ( +
+

+ Unplaced ({entries.length}) +

+

+ These Containers belong to this {collectionLabel} but have no grid position, or a stored position that is not + on the {collectionLabel} grid. +

+
+ + + + {['Stored position', 'Barcode', 'Subject or control', 'Specimen type', 'Status'].map((label) => ( + + ))} + + + + {entries.map((entry) => { + const container = entry.container as CollectionEntryContainer | null | undefined + const source = container?.source + const sourceName = source?.type === 'subject' || source?.type === 'control' ? (source.name ?? '') : '' + const status = + container == null + ? '' + : container.remainingQuantity != null && container.remainingQuantity > 0 + ? 'In Use' + : 'Exhausted' + return ( + + + + + + + + ) + })} + +
+ {label} +
+ {entry.position ? entry.position : None} + + + {entry.barcode || `Container ${entry.id}`} + + {sourceName}{container?.specimenTypeName ?? ''}{status}
+
+
+ ) +} diff --git a/packages/web/src/lib/api/collections.ts b/packages/web/src/lib/api/collections.ts index a8482df0..12d7c542 100644 --- a/packages/web/src/lib/api/collections.ts +++ b/packages/web/src/lib/api/collections.ts @@ -68,7 +68,7 @@ interface SheetResponse { updatedBy?: number | null } -interface WellEntry { +export interface WellEntry { type: 'micronix_tube' | 'static_well' id: number barcode?: string | null @@ -76,7 +76,7 @@ interface WellEntry { container?: unknown } -interface CryovialTubeEntry { +export interface CryovialTubeEntry { kind: 'cryovial_tube' id: number barcode?: string | null @@ -91,6 +91,22 @@ interface PaperEntry { container?: unknown } +export interface MicronixPlateDetail { + plate: MicronixPlateResponse + /** Placed Containers keyed by grid cell (A01–H12). */ + wells: Record + /** Containers with no grid position, or a stored position that is not a plate cell. */ + unplaced: WellEntry[] +} + +export interface CryovialBoxDetail { + box: CryovialBoxResponse + /** Placed Containers keyed by grid cell (A01–J10). */ + positions: Record + /** Containers with no grid position, or a stored position that is not a box cell. */ + unplaced: CryovialTubeEntry[] +} + export interface ValidatePlateScanResult { plate: { id: number; name: string } summary: { @@ -98,6 +114,8 @@ export interface ValidatePlateScanResult { matched: number missingInScan: number extraInScan: number + /** Unplaced tubes of this plate that the scan found at some position. */ + unplacedInScan: number mismatch: number exhaustedCount: number taggedCount: number @@ -106,12 +124,15 @@ export interface ValidatePlateScanResult { position: string scanBarcode: string | null expectedBarcode: string | null - status: 'match' | 'mismatch' | 'missing_in_scan' | 'extra_in_scan' + /** `unplaced_in_scan`: the scanned tube belongs to this plate but has no grid position. */ + status: 'match' | 'mismatch' | 'missing_in_scan' | 'extra_in_scan' | 'unplaced_in_scan' exhausted: boolean tags: string[] /** When status is mismatch or extra_in_scan, where the scanned barcode is registered (plate + position). */ scanBarcodeOrigin: { plateId: number; plateName: string; position: string } | null }> + /** Tubes in the plate with no grid position, or a stored position that is not a plate cell. */ + unplaced: Array<{ barcode: string; storedPosition: string | null; scannedPosition: string | null }> /** True when plate was inferred from scan barcodes (plateId was not sent). */ inferredPlate?: boolean } @@ -137,9 +158,9 @@ export type ValidatePlateScanResponse = export const collectionsApi = { getMicronixPlate: (id: number) => - api.get<{ plate: MicronixPlateResponse; wells: Record }>(`/collections/plates/micronix/${id}`), + api.get(`/collections/plates/micronix/${id}`), getCryovialBox: (id: number) => - api.get<{ box: CryovialBoxResponse; positions: Record }>(`/collections/boxes/cryovial/${id}`), + api.get(`/collections/boxes/cryovial/${id}`), getBox: (id: number) => api.get<{ box: BoxResponse; contents: { sheets: Array } }>(`/collections/boxes/${id}`), getBag: (id: number) => diff --git a/packages/web/src/pages/CryovialBoxDetail.tsx b/packages/web/src/pages/CryovialBoxDetail.tsx index 81311e3c..accb3c2c 100644 --- a/packages/web/src/pages/CryovialBoxDetail.tsx +++ b/packages/web/src/pages/CryovialBoxDetail.tsx @@ -15,6 +15,8 @@ import { import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { Link } from 'react-router-dom' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' +import UnplacedContainersSection from '../components/UnplacedContainersSection' +import { CRYOVIAL_BOX_GRID } from '@sampledb/contract' import '../styles/storage.css' function statusColor(name: string): string { @@ -59,29 +61,14 @@ export default function CryovialBoxDetail() { } }, [targetPosition, data]) - const layout = useMemo(() => { - if (!data?.positions) return null - const positions = Object.keys(data.positions) - const rowSet = new Set() - const colSet = new Set() - - positions.forEach((pos) => { - if (!pos) return - const match = pos.match(/^([A-Z]+)(\d+)$/i) - if (match) { - rowSet.add(match[1].toUpperCase()) - colSet.add(String(parseInt(match[2], 10))) - } - }) - - const rows = Array.from(rowSet).sort() - const cols = Array.from(colSet) - .map((c) => parseInt(c, 10)) - .sort((a, b) => a - b) - .map((n) => String(n)) - - return { rows, cols } - }, [data]) + // The grid always shows the whole box (rows A–J, columns 1–10), whatever it holds. + const layout = useMemo( + () => ({ + rows: [...CRYOVIAL_BOX_GRID.rows], + cols: CRYOVIAL_BOX_GRID.columns.map((column) => String(column)), + }), + [], + ) const legend = useMemo(() => { if (!data?.positions) return { tags: [] as string[], statuses: [] as string[] } @@ -108,7 +95,7 @@ export default function CryovialBoxDetail() { }, [data]) const tableRows = useMemo(() => { - if (!layout || !data?.positions) return [] + if (!data?.positions) return [] const positions = data.positions as Record const box = data.box const context = { collectionName: box.name, locationPath: box.locationPath ?? undefined } @@ -130,6 +117,18 @@ export default function CryovialBoxDetail() { }) }) }) + // Unplaced Containers follow the grid, labelled with their stored position. + for (const entry of data.unplaced as Array) { + rows.push( + buildCollectionTableRow({ + position: entry.position ?? '', + barcode: entry.barcode, + containerType: entry.kind ?? undefined, + container: entry.container ?? undefined, + context, + }) + ) + } return rows }, [data, layout]) @@ -178,7 +177,7 @@ export default function CryovialBoxDetail() { ) } - const { box, positions } = data + const { box, positions, unplaced } = data const breadcrumbItems = [ { label: 'Locations', to: '/locations' }, @@ -215,8 +214,7 @@ export default function CryovialBoxDetail() { )}
- {layout && ( -
+

Box Layout

@@ -389,6 +387,7 @@ export default function CryovialBoxDetail() { }} /> )} + {viewMode === 'grid' && } {viewMode === 'table' && ( )}
- )}
{ - // Always return the full 96-well plate format (8 rows × 12 columns) - const rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'] - const cols = ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] - return { rows, cols } - }, []) + const layout = useMemo( + () => ({ + rows: [...MICRONIX_PLATE_GRID.rows], + cols: MICRONIX_PLATE_GRID.columns.map((column) => String(column)), + }), + [], + ) const legend = useMemo(() => { if (!data?.wells) return { tags: [] as string[], statuses: [] as string[] } @@ -122,6 +125,18 @@ export default function MicronixPlateDetail() { } }) }) + // Unplaced Containers follow the grid, labelled with their stored position. + for (const entry of data.unplaced as Array) { + rows.push( + buildCollectionTableRow({ + position: entry.position ?? '', + barcode: entry.barcode, + containerType: entry.type ?? undefined, + container: entry.container ?? undefined, + context, + }) + ) + } return rows }, [data, layout]) @@ -169,7 +184,7 @@ export default function MicronixPlateDetail() { ) } - const { plate, wells } = data + const { plate, wells, unplaced } = data const breadcrumbItems = [ { label: 'Locations', to: '/locations' }, @@ -374,6 +389,7 @@ export default function MicronixPlateDetail() { }} /> )} + {viewMode === 'grid' && } {viewMode === 'table' && ( csvEscape(String(c))).join(',')) } + if (result.unplaced.length > 0) { + lines.push('') + lines.push('Unplaced tubes (no grid position in the plate)') + lines.push(['Barcode', 'Stored position', 'Scanned position'].map(csvEscape).join(',')) + for (const t of result.unplaced) { + lines.push([t.barcode, t.storedPosition ?? '', t.scannedPosition ?? ''].map(csvEscape).join(',')) + } + } return lines.join('\r\n') } @@ -583,7 +596,7 @@ export default function PlateScanValidation() { Download report
-
+
{result.summary.matched}
Matched
@@ -600,6 +613,10 @@ export default function PlateScanValidation() {
{result.summary.extraInScan}
Extra in scan
+
+
{result.summary.unplacedInScan}
+
Unplaced in plate
+
{result.inferredPlate && (

Plate was inferred from the barcodes on the scan.

@@ -617,6 +634,12 @@ export default function PlateScanValidation() { ? 'Scan matches the database.' : 'There are discrepancies. Review the well grid below.'}

+ {result.unplaced.length > 0 && ( +

+ {result.unplaced.length} {result.unplaced.length === 1 ? 'tube belongs' : 'tubes belong'} to this plate + without a grid position; {result.summary.unplacedInScan} found in the scan. +

+ )}
@@ -675,6 +698,35 @@ export default function PlateScanValidation() { )} + {result && result.unplaced.length > 0 && ( +
+

Unplaced tubes

+

+ These tubes belong to this plate but have no grid position, or a stored position that is not a plate cell. +

+
+ + + + + + + + + + {result.unplaced.map((t) => ( + + + + + + ))} + +
BarcodeStored positionScanned position
{t.barcode}{t.storedPosition ?? 'None'}{t.scannedPosition ?? 'Not in scan'}
+
+
+ )} + {inferenceReport && (
diff --git a/packages/web/src/pages/__tests__/CryovialBoxDetail.test.tsx b/packages/web/src/pages/__tests__/CryovialBoxDetail.test.tsx index 551a41c4..be9ec122 100644 --- a/packages/web/src/pages/__tests__/CryovialBoxDetail.test.tsx +++ b/packages/web/src/pages/__tests__/CryovialBoxDetail.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import userEvent from '@testing-library/user-event' import CryovialBoxDetail from '../CryovialBoxDetail' import * as csv from '../../lib/csv' @@ -61,9 +61,29 @@ describe('CryovialBoxDetail', () => { vi.mocked(collectionsApi.getCryovialBox).mockResolvedValue({ box: mockBox, positions: mockPositions, + unplaced: [], }) }) + it('shows an error when the box fails to load', async () => { + vi.mocked(collectionsApi.getCryovialBox).mockRejectedValue(new Error('boom')) + await render() + expect(await screen.findByText('Could not load cryovial box')).toBeInTheDocument() + expect(screen.queryByRole('heading', { name: /Unplaced/ })).not.toBeInTheDocument() + }) + + it('draws every row and column of the box in natural order, even when only A01 is filled', async () => { + await render() + await screen.findByText('Box Layout') + + const grid = screen.getAllByRole('table')[0]! + const headerCells = within(within(grid).getAllByRole('row')[0]!).getAllByRole('columnheader') + expect(headerCells.slice(1).map((c) => c.textContent)).toEqual(['1', '2', '3', '4', '5', '6', '7', '8', '9', '10']) + const rowLabels = within(grid).getAllByRole('row').slice(1).map((row) => row.firstElementChild?.textContent) + expect(rowLabels).toEqual(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']) + expect(within(grid).getAllByText('Empty')).toHaveLength(99) + }) + it('shows box layout and Grid/Table toggle', async () => { await render() await waitFor(() => { @@ -101,4 +121,62 @@ describe('CryovialBoxDetail', () => { expect(filename).toMatch(/cryovial-box.*\.csv$/) downloadSpy.mockRestore() }) + + describe('with unplaced Containers', () => { + const unplacedTube = (id: number, barcode: string, position: string | null) => ({ + kind: 'cryovial_tube' as const, + id, + barcode, + ...(position != null && { position }), + container: { specimenId: id, remainingQuantity: 0, tags: [], source: { type: 'subject', name: `Sub-${id}` } }, + }) + + beforeEach(() => { + vi.mocked(collectionsApi.getCryovialBox).mockResolvedValue({ + box: mockBox, + positions: mockPositions, + unplaced: [ + unplacedTube(401, 'CV-DASH', '--A01'), + unplacedTube(402, 'CV-NA', 'NA01'), + unplacedTube(403, 'CV-NOPOS', null), + ], + }) + }) + + it('lists unplaced Containers in an Unplaced section labelled with their stored position', async () => { + await render() + + const heading = await screen.findByRole('heading', { name: 'Unplaced (3)' }) + const section = heading.closest('section')! + const rows = within(section).getAllByRole('row').slice(1) + expect(rows.map((row) => within(row).getAllByRole('cell').map((c) => c.textContent))).toEqual([ + ['--A01', 'CV-DASH', 'Sub-401', '', 'Exhausted'], + ['NA01', 'CV-NA', 'Sub-402', '', 'Exhausted'], + ['None', 'CV-NOPOS', 'Sub-403', '', 'Exhausted'], + ]) + }) + + it('includes unplaced Containers in the table view and its export', async () => { + const downloadSpy = vi.spyOn(csv, 'downloadCsv').mockImplementation(() => {}) + await render() + await screen.findByText('Box Layout') + await userEvent.click(screen.getByRole('button', { name: /Table/i })) + + const table = await screen.findByRole('table') + const rows = within(table).getAllByRole('row').slice(1) + expect(rows.map((row) => within(row).getAllByRole('cell').slice(0, 2).map((c) => c.textContent))).toEqual([ + ['A01', 'CV-001'], + ['--A01', 'CV-DASH'], + ['NA01', 'CV-NA'], + ['', 'CV-NOPOS'], + ]) + + await userEvent.click(screen.getByRole('button', { name: /Export CSV/i })) + const [csvContent] = downloadSpy.mock.calls[0]! + expect(csvContent).toContain('CV-DASH') + expect(csvContent).toContain('NA01') + expect(csvContent).toContain('CV-NOPOS') + downloadSpy.mockRestore() + }) + }) }) diff --git a/packages/web/src/pages/__tests__/MicronixPlateDetail.test.tsx b/packages/web/src/pages/__tests__/MicronixPlateDetail.test.tsx index 7e2548a6..262dc76c 100644 --- a/packages/web/src/pages/__tests__/MicronixPlateDetail.test.tsx +++ b/packages/web/src/pages/__tests__/MicronixPlateDetail.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import userEvent from '@testing-library/user-event' import MicronixPlateDetail from '../MicronixPlateDetail' import * as csv from '../../lib/csv' @@ -60,9 +60,17 @@ describe('MicronixPlateDetail', () => { vi.mocked(collectionsApi.getMicronixPlate).mockResolvedValue({ plate: mockPlate, wells: mockWells, + unplaced: [], }) }) + it('shows a loading state before the plate arrives', async () => { + vi.mocked(collectionsApi.getMicronixPlate).mockReturnValue(new Promise(() => {})) + await render() + expect(screen.queryByText('Plate Layout')).not.toBeInTheDocument() + expect(screen.queryByRole('heading', { name: /Unplaced/ })).not.toBeInTheDocument() + }) + it('shows plate layout and loads grid by default', async () => { await render() await waitFor(() => { @@ -147,4 +155,72 @@ describe('MicronixPlateDetail', () => { expect(screen.queryByRole('columnheader', { name: /Subject Name/i })).not.toBeInTheDocument() expect(screen.queryByRole('columnheader', { name: /Study Code/i })).not.toBeInTheDocument() }) + + describe('with unplaced Containers', () => { + const unplacedTube = (id: number, barcode: string, position: string | null) => ({ + type: 'micronix_tube' as const, + id, + barcode, + ...(position != null && { position }), + container: { specimenId: id, remainingQuantity: 1, tags: [], source: { type: 'subject', name: `Sub-${id}` } }, + }) + + beforeEach(() => { + vi.mocked(collectionsApi.getMicronixPlate).mockResolvedValue({ + plate: mockPlate, + wells: mockWells, + unplaced: [ + unplacedTube(301, 'MTX-BAD', 'A1145868849'), + unplacedTube(302, 'MTX-NOPOS-1', null), + unplacedTube(303, 'MTX-NOPOS-2', null), + ], + }) + }) + + it('lists unplaced Containers in an Unplaced section labelled with their stored position', async () => { + await render() + + const heading = await screen.findByRole('heading', { name: 'Unplaced (3)' }) + const section = heading.closest('section')! + const rows = within(section).getAllByRole('row').slice(1) + expect(rows.map((row) => within(row).getAllByRole('cell')[0]!.textContent)).toEqual([ + 'A1145868849', + 'None', + 'None', + ]) + expect(within(section).getByRole('link', { name: 'MTX-NOPOS-1' })).toHaveAttribute('href', '/containers/302') + expect(within(section).getByText('Sub-303')).toBeInTheDocument() + }) + + it('includes unplaced Containers after the 96 wells in the table view', async () => { + await render() + await screen.findByText('Plate Layout') + await userEvent.click(screen.getByRole('button', { name: /Table/i })) + + const table = await screen.findByRole('table') + const rows = within(table).getAllByRole('row').slice(1) + expect(rows).toHaveLength(99) + expect(rows.slice(96).map((row) => within(row).getAllByRole('cell').map((c) => c.textContent))).toEqual([ + ['In Use', 'A1145868849', 'MTX-BAD'], + ['In Use', '', 'MTX-NOPOS-1'], + ['In Use', '', 'MTX-NOPOS-2'], + ]) + expect(screen.queryByRole('heading', { name: /Unplaced/ })).not.toBeInTheDocument() + }) + + it('exports unplaced Containers with the table', async () => { + const downloadSpy = vi.spyOn(csv, 'downloadCsv').mockImplementation(() => {}) + await render() + await screen.findByText('Plate Layout') + await userEvent.click(screen.getByRole('button', { name: /Table/i })) + await userEvent.click(await screen.findByRole('button', { name: /Export CSV/i })) + + const [csvContent] = downloadSpy.mock.calls[0]! + expect(csvContent).toContain('A1145868849') + expect(csvContent).toContain('MTX-BAD') + expect(csvContent).toContain('MTX-NOPOS-1') + expect(csvContent).toContain('MTX-NOPOS-2') + downloadSpy.mockRestore() + }) + }) }) diff --git a/packages/web/src/pages/__tests__/PlateScanValidation.test.tsx b/packages/web/src/pages/__tests__/PlateScanValidation.test.tsx index c25b27e3..22dd4bb2 100644 --- a/packages/web/src/pages/__tests__/PlateScanValidation.test.tsx +++ b/packages/web/src/pages/__tests__/PlateScanValidation.test.tsx @@ -1,6 +1,7 @@ import { screen, fireEvent, waitFor } from '@testing-library/react' import { vi, describe, it, expect, beforeEach } from 'vitest' -import { renderWithProviders } from '../../__tests__/helpers/render' +import { renderWithProviders, render, within } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' import PlateScanValidation from '../PlateScanValidation' import { collectionsApi } from '../../lib/api/collections' import { settingsApi } from '../../lib/api/settings' @@ -49,11 +50,13 @@ const mockValidateResult = { matched: 96, missingInScan: 0, extraInScan: 0, + unplacedInScan: 0, mismatch: 0, exhaustedCount: 0, taggedCount: 0, }, wells: [], + unplaced: [], } describe('PlateScanValidation', () => { @@ -276,4 +279,55 @@ describe('PlateScanValidation', () => { expect(screen.getByText(/Cannot infer plate/i)).toBeInTheDocument() }, { timeout: 5000 }) }) + + it('shows scanned unplaced tubes as belonging to the plate, not as extras', async () => { + const user = userEvent.setup() + const csvContent = 'Well,Barcode\nA01,MT-PLACED\nB01,MT-NULL-1\nC01,MT-NULL-2' + const file = new File([csvContent], 'scan.csv', { type: 'text/csv' }) + Object.defineProperty(file, 'text', { value: async () => csvContent }) + vi.mocked(collectionsApi.validatePlateScan).mockResolvedValue({ + plate: { id: 1, name: 'PLATE1' }, + summary: { + totalExpected: 4, + matched: 1, + missingInScan: 1, + extraInScan: 0, + unplacedInScan: 2, + mismatch: 0, + exhaustedCount: 0, + taggedCount: 0, + }, + wells: [ + { position: 'A01', scanBarcode: 'MT-PLACED', expectedBarcode: 'MT-PLACED', status: 'match', exhausted: false, tags: [], scanBarcodeOrigin: null }, + { position: 'B01', scanBarcode: 'MT-NULL-1', expectedBarcode: null, status: 'unplaced_in_scan', exhausted: false, tags: [], scanBarcodeOrigin: { plateId: 1, plateName: 'PLATE1', position: '' } }, + { position: 'C01', scanBarcode: 'MT-NULL-2', expectedBarcode: null, status: 'unplaced_in_scan', exhausted: false, tags: [], scanBarcodeOrigin: { plateId: 1, plateName: 'PLATE1', position: '' } }, + ], + unplaced: [ + { barcode: 'MT-NULL-1', storedPosition: null, scannedPosition: 'B01' }, + { barcode: 'MT-NULL-2', storedPosition: null, scannedPosition: 'C01' }, + { barcode: 'MT-BAD-POS', storedPosition: 'A1145868849', scannedPosition: null }, + ], + }) + + const { container } = await render() + await screen.findByRole('heading', { name: 'Validate Plate Scan' }) + await user.click(screen.getByRole('radio', { name: /infer plate from scan/i })) + await user.upload(container.querySelector('input[type="file"]') as HTMLInputElement, file) + await screen.findByText('scan.csv') + await user.click(screen.getByRole('button', { name: /Validate scan/i })) + + const unplacedHeading = await screen.findByRole('heading', { name: 'Unplaced tubes' }) + const wellGrid = screen.getByRole('heading', { name: 'Well grid' }).parentElement! + const statuses = within(wellGrid).getAllByRole('row').slice(1).map((row) => within(row).getAllByRole('cell')[4]!.textContent) + expect(statuses).toEqual(['Match', 'Unplaced in plate', 'Unplaced in plate']) + expect(within(wellGrid).queryByText('Extra in scan')).not.toBeInTheDocument() + + const unplacedRows = within(unplacedHeading.parentElement!).getAllByRole('row').slice(1) + expect(unplacedRows.map((row) => within(row).getAllByRole('cell').map((c) => c.textContent))).toEqual([ + ['MT-NULL-1', 'None', 'B01'], + ['MT-NULL-2', 'None', 'C01'], + ['MT-BAD-POS', 'A1145868849', 'Not in scan'], + ]) + expect(screen.getByText(/3 tubes belong to this plate without a grid position; 2 found in the scan/)).toBeInTheDocument() + }) }) From b9215137a1dca685de11a2e622b896b2a5dac8a1 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:34:05 -0700 Subject: [PATCH 07/54] fix(security): hash session tokens, cap error-log intake, generic production 500s (#121) Sessions - The session cookie is now a random 256-bit token stored only as its SHA-256 hash (sessions.token_hash, unique index). sessions.id is a separate public id that admins list and revoke by; it cannot authenticate. Cookie lookups stay one indexed equality on token_hash. - Session create/lookup/delete moves into lib/auth/auth-service.ts (login, logout, switch user, password change and reset, user delete, auth middleware all use it). - Migration 009 (009_session_token_hash.sql) drops and recreates the sessions table. Existing sessions stored the cookie value in id and SQLite cannot hash it in SQL, so they are invalidated: every user logs in again once after this release. Noted in the user-guide release notes. - CURRENT_SCHEMA_VERSION goes 7 -> 9; 008 is reserved for another change and must land before any database reaches 9. Error-log intake - POST /api/error-logs takes a 64 KiB body cap (413, PAYLOAD_TOO_LARGE, not persisted) and per-field caps (message 2,000, stack 16,000, errorCode 200, context 16,000 chars of JSON; 400 VALIDATION_ERROR). The caps and schema live in @sampledb/contract (error-report.ts); the web error logger cuts its reports to fit before posting. Error messages - With NODE_ENV=production a 500 from handleRouteError returns {"error":"Internal server error","errorCode":"INTERNAL_ERROR"}; the full error goes to stdout (observability) and error_logs. containersFetchFailedBody drops the cause's message in production too. Co-Authored-By: Claude Opus 5.5 --- packages/api/initial_schema.sql | 5 +- .../api/src/__tests__/helpers/auth-helpers.ts | 10 +- .../src/db/__tests__/schema-evolution.test.ts | 32 ++++ .../db/migrations/009_session_token_hash.sql | 13 ++ packages/api/src/db/schema-evolution.ts | 2 +- packages/api/src/db/schema.ts | 8 +- .../src/lib/__tests__/error-handler.test.ts | 74 ++++++++++ packages/api/src/lib/auth/auth-service.ts | 69 ++++++++- packages/api/src/lib/error-handler.ts | 52 ++++++- packages/api/src/middleware/auth.ts | 22 +-- .../routes/__tests__/auth-sessions.test.ts | 138 ++++++++++++++++++ .../api/src/routes/__tests__/auth.test.ts | 6 +- .../src/routes/__tests__/error-logs.test.ts | 70 +++++++++ packages/api/src/routes/auth.ts | 76 ++++------ packages/api/src/routes/error-logs.ts | 32 ++-- .../src/__tests__/error-report.test.ts | 41 ++++++ packages/contract/src/error-report.ts | 66 +++++++++ packages/contract/src/index.ts | 6 + .../guides/troubleshooting/release-notes.md | 10 ++ .../src/lib/__tests__/error-logger.test.ts | 21 +++ packages/web/src/lib/api/admin.ts | 5 +- packages/web/src/lib/api/auth.ts | 1 + packages/web/src/lib/error-logger.ts | 33 +++-- .../src/pages/__tests__/AdminUsers.test.tsx | 21 +++ 24 files changed, 700 insertions(+), 113 deletions(-) create mode 100644 packages/api/src/db/migrations/009_session_token_hash.sql create mode 100644 packages/api/src/routes/__tests__/auth-sessions.test.ts create mode 100644 packages/contract/src/__tests__/error-report.test.ts create mode 100644 packages/contract/src/error-report.ts diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index ce2b56e2..a1d39679 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -13,9 +13,12 @@ CREATE TABLE IF NOT EXISTS users ( );--> statement-breakpoint CREATE TABLE IF NOT EXISTS sessions ( id TEXT PRIMARY KEY, + token_hash TEXT NOT NULL, user_id INTEGER NOT NULL REFERENCES users(id), expires_at INTEGER NOT NULL );--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS sessions_token_hash_idx ON sessions(token_hash);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS sessions_user_id_idx ON sessions(user_id);--> statement-breakpoint CREATE TABLE IF NOT EXISTS tag ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE @@ -284,7 +287,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (7);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (9);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/__tests__/helpers/auth-helpers.ts b/packages/api/src/__tests__/helpers/auth-helpers.ts index f46ffd54..99e3c549 100644 --- a/packages/api/src/__tests__/helpers/auth-helpers.ts +++ b/packages/api/src/__tests__/helpers/auth-helpers.ts @@ -1,7 +1,7 @@ import type { Database } from '../../db/client' import { users, sessions, settings } from '../../db/schema' import bcrypt from 'bcryptjs' -import { BCRYPT_COST } from '../../lib/auth/auth-service' +import { BCRYPT_COST, hashSessionToken } from '../../lib/auth/auth-service' import { nanoid } from 'nanoid' import type { Hono } from 'hono' import { testClient } from 'hono/testing' @@ -150,11 +150,14 @@ export async function createAuthenticatedSession( userId: number, maxAgeSeconds: number = 604800 ) { - const sessionId = nanoid() + // sessionId is the cookie token; the row stores only its hash, under a separate public id. + const sessionId = nanoid(43) + const publicId = nanoid() const expiresAt = Math.floor(Date.now() / 1000) + maxAgeSeconds await db.insert(sessions).values({ - id: sessionId, + id: publicId, + tokenHash: hashSessionToken(sessionId), userId, expiresAt, }) @@ -162,6 +165,7 @@ export async function createAuthenticatedSession( // Return session cookie in format expected by Hono test client return { sessionId, + publicId, cookie: `session_id=${sessionId}`, cookieHeader: `session_id=${sessionId}; Path=/; HttpOnly; SameSite=Lax`, } diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index 65caa5ae..ab320cc6 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -42,6 +42,12 @@ function createTablesForLaterMigrations(sqlite: Database): void { box_id INTEGER, bag_id INTEGER )`) + sqlite.exec(`CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, email TEXT NOT NULL, created TEXT, approved_at TEXT)`) + sqlite.exec(`CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + expires_at INTEGER NOT NULL + )`) } describe('schema evolution', () => { @@ -340,6 +346,32 @@ describe('schema evolution', () => { sqlite.close() }) + it('migration 009 drops sessions stored by cookie value and stores sessions by token hash', () => { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (7)') + createTablesForLaterMigrations(sqlite) + sqlite.exec(`INSERT INTO users (id, email) VALUES (1, 'a@test.com')`) + sqlite.exec(`INSERT INTO sessions (id, user_id, expires_at) VALUES ('raw-cookie-value', 1, 9999999999)`) + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(sqlite.prepare('SELECT COUNT(*) AS n FROM sessions').get()).toEqual({ n: 0 }) + const columns = (sqlite.prepare('PRAGMA table_info(sessions)').all() as Array<{ name: string; notnull: number }>) + .map((column) => `${column.name}${column.notnull ? ' NOT NULL' : ''}`) + expect(columns).toEqual(['id', 'token_hash NOT NULL', 'user_id NOT NULL', 'expires_at NOT NULL']) + sqlite.exec(`INSERT INTO sessions (id, token_hash, user_id, expires_at) VALUES ('p1', 'h1', 1, 1)`) + expect(() => + sqlite.exec(`INSERT INTO sessions (id, token_hash, user_id, expires_at) VALUES ('p2', 'h1', 1, 1)`), + ).toThrow(/UNIQUE/) + const plan = sqlite + .prepare(`EXPLAIN QUERY PLAN SELECT id FROM sessions WHERE token_hash = 'h1'`) + .all() as Array<{ detail: string }> + expect(plan.map((row) => row.detail).join(' ')).toContain('sessions_token_hash_idx') + sqlite.close() + }) + it('fail-hard: invalid migration leaves schema_version unchanged', () => { const sqlite = new Database(testDbPath) sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') diff --git a/packages/api/src/db/migrations/009_session_token_hash.sql b/packages/api/src/db/migrations/009_session_token_hash.sql new file mode 100644 index 00000000..6aeefeb5 --- /dev/null +++ b/packages/api/src/db/migrations/009_session_token_hash.sql @@ -0,0 +1,13 @@ +-- Migration 009: a session's cookie token is stored only as its SHA-256 hash (token_hash), and +-- sessions.id becomes a public id that admins list and revoke by, which cannot authenticate. +-- Existing rows stored the cookie value itself in id, and SQLite cannot hash it here, so they are +-- dropped: every user logs in again after this release. +DROP TABLE IF EXISTS sessions;--> statement-breakpoint +CREATE TABLE sessions ( + id TEXT PRIMARY KEY, + token_hash TEXT NOT NULL, + user_id INTEGER NOT NULL REFERENCES users(id), + expires_at INTEGER NOT NULL +);--> statement-breakpoint +CREATE UNIQUE INDEX sessions_token_hash_idx ON sessions(token_hash);--> statement-breakpoint +CREATE INDEX sessions_user_id_idx ON sessions(user_id); diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index b96368bd..c317050d 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -7,7 +7,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 7 +export const CURRENT_SCHEMA_VERSION = 9 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/db/schema.ts b/packages/api/src/db/schema.ts index 8b339eab..9ff59d80 100644 --- a/packages/api/src/db/schema.ts +++ b/packages/api/src/db/schema.ts @@ -55,11 +55,17 @@ export const users = sqliteTable('users', { approvedAt: text('approved_at'), // null = pending approval; non-null = approved }) +// A session's cookie value is a random token stored only as its SHA-256 hash in token_hash. +// id is a public id for listing and revoking; it does not authenticate. export const sessions = sqliteTable('sessions', { id: text('id').primaryKey(), + tokenHash: text('token_hash').notNull(), userId: integer('user_id').notNull().references(() => users.id), expiresAt: integer('expires_at').notNull(), -}) +}, (table) => ({ + tokenHashIdx: uniqueIndex('sessions_token_hash_idx').on(table.tokenHash), + userIdIdx: index('sessions_user_id_idx').on(table.userId), +})) // Tags and Units (Standardized System) export const tag = sqliteTable('tag', { diff --git a/packages/api/src/lib/__tests__/error-handler.test.ts b/packages/api/src/lib/__tests__/error-handler.test.ts index df8af841..c26c98c8 100644 --- a/packages/api/src/lib/__tests__/error-handler.test.ts +++ b/packages/api/src/lib/__tests__/error-handler.test.ts @@ -196,4 +196,78 @@ describe('error-handler', () => { expect(data.details).toBe('timeout') }) }) + + describe('with NODE_ENV=production', () => { + const sqlMessage = 'SQLITE_ERROR: no such column: storage_container.secret_col in SELECT * FROM storage_container' + let previousNodeEnv: string | undefined + let previousLogFormat: string | undefined + let originalConsoleError: typeof console.error + let stderrLines: string[] + + beforeEach(() => { + previousNodeEnv = process.env.NODE_ENV + previousLogFormat = process.env.LOG_FORMAT + process.env.NODE_ENV = 'production' + process.env.LOG_FORMAT = 'json' + stderrLines = [] + originalConsoleError = console.error + console.error = (...args: unknown[]) => { + stderrLines.push(args.map(String).join(' ')) + } + }) + + afterEach(() => { + console.error = originalConsoleError + process.env.NODE_ENV = previousNodeEnv + if (previousLogFormat === undefined) delete process.env.LOG_FORMAT + else process.env.LOG_FORMAT = previousLogFormat + }) + + it('answers a 500 with a generic message and still logs the full error', async () => { + const app = createAppWithError(new Error(sqlMessage)) + const res = await app.request('http://localhost/test') + + expect(res.status).toBe(500) + const text = await res.text() + expect(JSON.parse(text)).toEqual({ error: 'Internal server error', errorCode: 'INTERNAL_ERROR' }) + expect(text).not.toContain('SQLITE') + expect(text).not.toContain('storage_container') + + // stdout/stderr gets the full error + expect(stderrLines.some((line) => line.includes(sqlMessage))).toBe(true) + // error_logs gets the full error, with its stack + await new Promise((resolve) => setTimeout(resolve, 50)) + const rows = await testDb.select().from(errorLogs).where(eq(errorLogs.message, sqlMessage)) + expect(rows).toHaveLength(1) + expect(rows[0].level).toBe('error') + expect(rows[0].stack).toContain(sqlMessage) + }) + + it('answers a 500 for a non-Error thrown value with a generic message and no details', async () => { + const res = handleRouteError(sqlMessage, createMockContext()) + expect(res.status).toBe(500) + expect(await getJson(res)).toEqual({ error: 'Internal server error', errorCode: 'INTERNAL_ERROR' }) + expect(stderrLines.some((line) => line.includes(sqlMessage))).toBe(true) + }) + + it('keeps the cause out of the containers and search 500 bodies', async () => { + const containers = createAppWithError( + new RouteError(500, containersFetchFailedBody('Failed to fetch containers', new Error(sqlMessage))) + ) + const containersRes = await containers.request('http://localhost/test') + expect(containersRes.status).toBe(500) + expect(await getJson(containersRes)).toEqual({ error: 'Failed to fetch containers' }) + + const search = createAppWithError(new RouteError(500, searchFailedBody('flu', new Error(sqlMessage)))) + const searchText = await (await search.request('http://localhost/test')).text() + expect(searchText).not.toContain(sqlMessage) + }) + + it('keeps the raw SQLite text out of a constraint-conflict 409', async () => { + const error = Object.assign(new Error('UNIQUE constraint failed: study.title'), { code: 'SQLITE_CONSTRAINT_UNIQUE' }) + const res = await createAppWithError(error).request('http://localhost/test') + expect(res.status).toBe(409) + expect(await res.text()).not.toContain('UNIQUE constraint failed') + }) + }) }) diff --git a/packages/api/src/lib/auth/auth-service.ts b/packages/api/src/lib/auth/auth-service.ts index c924ddf1..7b4663d7 100644 --- a/packages/api/src/lib/auth/auth-service.ts +++ b/packages/api/src/lib/auth/auth-service.ts @@ -1,4 +1,5 @@ -import { eq, and, isNull } from 'drizzle-orm' +import { eq, and, gt, isNull, ne } from 'drizzle-orm' +import { createHash, randomBytes } from 'node:crypto' import bcrypt from 'bcryptjs' import { nanoid } from 'nanoid' import type { Database } from '../../db/client' @@ -77,23 +78,77 @@ export async function verifyLoginCredentials( return user } +/** + * SHA-256 of a session cookie token, hex encoded. The sessions table stores only this hash, so a + * copy of the database or an admin session listing cannot be replayed as a cookie. The token has + * 256 bits of entropy, so an unsalted fast hash is enough and keeps the lookup an indexed equality. + */ +export function hashSessionToken(token: string): string { + return createHash('sha256').update(token).digest('hex') +} + +/** + * Start a session for a user. Returns the cookie token, which is not stored, and the session's + * public id, which admins list and revoke by. + */ export async function createUserSession( database: Database, - userId: number -): Promise<{ sessionId: string; expiresAt: number; maxAgeSeconds: number }> { - const sessionSettings = await getSessionSettings(database) + userId: number, + settingsDatabase: Database = database +): Promise<{ sessionToken: string; publicId: string; expiresAt: number; maxAgeSeconds: number }> { + const sessionSettings = await getSessionSettings(settingsDatabase) if (!sessionSettings) { throw new Error('Session settings are not configured. Please run database initialization.') } const maxAgeSeconds = sessionSettings.maxAgeSeconds - const sessionId = nanoid() + const sessionToken = randomBytes(32).toString('base64url') + const publicId = nanoid() const expiresAt = Math.floor(Date.now() / 1000) + maxAgeSeconds await database.insert(sessions).values({ - id: sessionId, + id: publicId, + tokenHash: hashSessionToken(sessionToken), userId, expiresAt, }) - return { sessionId, expiresAt, maxAgeSeconds } + return { sessionToken, publicId, expiresAt, maxAgeSeconds } +} + +/** The unexpired session a cookie token belongs to, or null. */ +export async function findActiveSessionByToken( + database: Database, + sessionToken: string +): Promise<{ id: string; userId: number; expiresAt: number } | null> { + return ( + (await database + .select({ id: sessions.id, userId: sessions.userId, expiresAt: sessions.expiresAt }) + .from(sessions) + .where(and( + eq(sessions.tokenHash, hashSessionToken(sessionToken)), + gt(sessions.expiresAt, Math.floor(Date.now() / 1000)) + )) + .get()) ?? null + ) +} + +/** End the session a cookie token belongs to, if any. */ +export async function deleteSessionByToken(database: Database, sessionToken: string): Promise { + await database.delete(sessions).where(eq(sessions.tokenHash, hashSessionToken(sessionToken))) +} + +/** + * End all of a user's sessions, or all but the one a cookie token belongs to when `keepToken` + * is given. + */ +export async function deleteUserSessions( + database: Database, + userId: number, + keepToken?: string +): Promise { + await database + .delete(sessions) + .where(keepToken + ? and(eq(sessions.userId, userId), ne(sessions.tokenHash, hashSessionToken(keepToken))) + : eq(sessions.userId, userId)) } export async function registerApprovedUser( diff --git a/packages/api/src/lib/error-handler.ts b/packages/api/src/lib/error-handler.ts index dd4282b1..bff3fff1 100644 --- a/packages/api/src/lib/error-handler.ts +++ b/packages/api/src/lib/error-handler.ts @@ -3,7 +3,7 @@ import type { ContentfulStatusCode } from 'hono/utils/http-status' import { z } from 'zod' import { logBackendError, logError, type ErrorLogContext } from './error-logger' import { getRequestDatabase } from './db-context' -import { logError as logObservabilityError } from './observability' +import { logError as logObservabilityError, logWarn } from './observability' import { getRequestId } from '../middleware/request-context' function logPersistFailure(logErr: unknown): void { @@ -48,6 +48,18 @@ export function sqliteConstraintConflict(error: Error): string | null { return null } +/** The `error` a client gets for a 500 in production. */ +export const INTERNAL_ERROR_MESSAGE = 'Internal server error' + +function logInternalError(error: unknown, c: Context): void { + const err = error instanceof Error ? error : new Error(String(error)) + logObservabilityError('Unhandled error in request', err, { + requestId: getRequestId(c), + method: c.req.method, + path: new URL(c.req.url).pathname, + }) +} + export function handleRouteError(error: unknown, c: Context): Response { const database = getRequestDatabase(c) @@ -104,6 +116,17 @@ export function handleRouteError(error: unknown, c: Context): Response { }, 401) } + // Not persisted to error_logs: an oversized body is rejected so it cannot fill the table. + // hono's bodyLimit raises BodyLimitError (not exported) when it finds the excess while the + // route reads a body sent without Content-Length. + if (error instanceof PayloadTooLargeError || error.name === 'BodyLimitError') { + logWarn('Request body too large', { requestId: getRequestId(c), method: c.req.method, path: new URL(url).pathname }) + return c.json({ + error: error instanceof PayloadTooLargeError ? error.message : 'Request body too large', + errorCode: 'PAYLOAD_TOO_LARGE', + }, 413) + } + if (error instanceof ExpectedNotFoundError) { return c.json({ error: error.message, @@ -164,24 +187,27 @@ export function handleRouteError(error: unknown, c: Context): Response { }, 409) } - // Log error asynchronously (non-blocking) + // Log the full error (stdout and error_logs); in production the client gets only a generic + // message, since error.message can carry SQL or other internals. + logInternalError(error, c) logBackendError(database, error, errorContext).catch(logPersistFailure) + if (!isDevelopment) { + return c.json({ error: INTERNAL_ERROR_MESSAGE, errorCode: 'INTERNAL_ERROR' }, 500) + } return c.json({ error: error.message, errorCode: error.name || 'APPLICATION_ERROR', - ...(isDevelopment && { - stack: error.stack, - details: error.cause - }) + stack: error.stack, + details: error.cause, }, 500) } // Handle unknown errors - // Log error asynchronously (non-blocking) + logInternalError(error, c) logBackendError(database, error, errorContext).catch(logPersistFailure) const isDevelopment = process.env.NODE_ENV !== 'production' return c.json({ - error: 'Internal server error', + error: INTERNAL_ERROR_MESSAGE, errorCode: 'INTERNAL_ERROR', ...(isDevelopment && { details: String(error) @@ -247,6 +273,14 @@ export type CollectionDeleteBlockerCode = | 'qpcr_wells_link_specimens' | 'container_derivation_spans_outside_collection' +/** 413: the request body is over a route's size cap. */ +export class PayloadTooLargeError extends Error { + constructor(message: string = 'Request body too large') { + super(message) + this.name = 'PayloadTooLargeError' + } +} + export class ValidationError extends Error { constructor(message: string, public details?: any) { super(message) @@ -292,6 +326,8 @@ function containersFetchFailedBody( message: string, cause: unknown, ): ErrorResponse & Record { + // The cause's message stays out of production responses, as in handleRouteError's 500s. + if (process.env.NODE_ENV === 'production') return { error: message } const errorMessage = cause instanceof Error ? cause.message : 'Unknown error' return { error: message, details: errorMessage } } diff --git a/packages/api/src/middleware/auth.ts b/packages/api/src/middleware/auth.ts index eca53e14..716e6af2 100644 --- a/packages/api/src/middleware/auth.ts +++ b/packages/api/src/middleware/auth.ts @@ -1,9 +1,10 @@ import { Context, Next } from 'hono' import { getCookie, setCookie, deleteCookie } from 'hono/cookie' -import { sessions, users } from '../db/schema' -import { eq, and, gt, isNull } from 'drizzle-orm' +import { users } from '../db/schema' +import { eq, and, isNull } from 'drizzle-orm' import type { Database } from '../db/client' import { UnauthorizedError } from '../lib/error-handler' +import { findActiveSessionByToken } from '../lib/auth/auth-service' export interface AuthUser { id: number @@ -25,24 +26,13 @@ declare module 'hono' { export function createAuthMiddleware(database: Database) { return async (c: Context, next: Next) => { const dbToUse = database - const sessionId = getCookie(c, 'session_id') + const sessionToken = getCookie(c, 'session_id') - if (!sessionId) { + if (!sessionToken) { throw new UnauthorizedError('No session ID provided') } - const session = await dbToUse - .select({ - id: sessions.id, - userId: sessions.userId, - expiresAt: sessions.expiresAt, - }) - .from(sessions) - .where(and( - eq(sessions.id, sessionId), - gt(sessions.expiresAt, Math.floor(Date.now() / 1000)) - )) - .get() + const session = await findActiveSessionByToken(dbToUse, sessionToken) if (!session) { deleteCookie(c, 'session_id') diff --git a/packages/api/src/routes/__tests__/auth-sessions.test.ts b/packages/api/src/routes/__tests__/auth-sessions.test.ts new file mode 100644 index 00000000..cec76830 --- /dev/null +++ b/packages/api/src/routes/__tests__/auth-sessions.test.ts @@ -0,0 +1,138 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { eq } from 'drizzle-orm' +import { + setupAuthenticatedRouteTest, + type AuthenticatedRouteTestContext, +} from '../../__tests__/helpers/authenticated-route-test' +import { createAuthRoutes } from '../auth' +import { sessions, users } from '../../db/schema' +import { hashSessionToken } from '../../lib/auth/auth-service' + +function cookieToken(cookie: string): string { + const match = cookie.match(/session_id=([^;]+)/) + if (!match) throw new Error(`No session_id in ${cookie}`) + return match[1] +} + +describe('Session tokens', () => { + let ctx: AuthenticatedRouteTestContext + let memberId: number + + beforeEach(async () => { + ctx = await setupAuthenticatedRouteTest({ + user: { email: 'admin@test.com', role: 'admin' }, + additionalUsers: [{ key: 'member', email: 'member@test.com', role: 'member' }], + mount: (app, { db }) => { + app.route('/api/auth', createAuthRoutes(db, db)) + }, + }) + const member = await ctx.db.select().from(users).where(eq(users.email, 'member@test.com')).get() + memberId = member!.id + }) + + afterEach(() => ctx.cleanup()) + + it('stores the SHA-256 hash of the cookie token, never the token itself', async () => { + const token = cookieToken(ctx.cookies.member) + const rows = await ctx.db.select().from(sessions).where(eq(sessions.userId, memberId)).all() + + expect(rows).toHaveLength(1) + expect(rows[0].tokenHash).toBe(hashSessionToken(token)) + expect(rows[0].tokenHash).not.toBe(token) + expect(rows[0].id).not.toBe(token) + expect(JSON.stringify(rows)).not.toContain(token) + }) + + it('lists a user\'s sessions by public id with no value that works as a session cookie', async () => { + const res = await ctx.request(`/api/auth/users/${memberId}/sessions`) + expect(res.status).toBe(200) + const body = (await res.json()) as { sessions: Array> } + + expect(body.sessions).toHaveLength(1) + expect(Object.keys(body.sessions[0]).sort()).toEqual(['expiresAt', 'id']) + const token = cookieToken(ctx.cookies.member) + expect(JSON.stringify(body)).not.toContain(token) + expect(JSON.stringify(body)).not.toContain(hashSessionToken(token)) + + // The listed id does not authenticate as the user. + const asCookie = await ctx.request('/api/auth/me', { cookie: `session_id=${String(body.sessions[0].id)}` }) + expect(asCookie.status).toBe(401) + }) + + it('revokes a session by its public id, after which its cookie no longer authenticates', async () => { + const before = await ctx.request('/api/auth/me', { cookie: ctx.cookies.member }) + expect(before.status).toBe(200) + + const list = (await (await ctx.request(`/api/auth/users/${memberId}/sessions`)).json()) as { + sessions: Array<{ id: string }> + } + const res = await ctx.request(`/api/auth/sessions/${list.sessions[0].id}`, { method: 'DELETE' }) + expect(res.status).toBe(200) + + const after = await ctx.request('/api/auth/me', { cookie: ctx.cookies.member }) + expect(after.status).toBe(401) + const remaining = await ctx.db.select().from(sessions).where(eq(sessions.userId, memberId)).all() + expect(remaining).toHaveLength(0) + }) + + it('does not revoke a session when given its cookie token instead of its public id', async () => { + const token = cookieToken(ctx.cookies.member) + const res = await ctx.request(`/api/auth/sessions/${token}`, { method: 'DELETE' }) + expect(res.status).toBe(404) + + const after = await ctx.request('/api/auth/me', { cookie: ctx.cookies.member }) + expect(after.status).toBe(200) + }) + + it('returns 404 for an unknown user\'s sessions and 400 for a non-numeric user id', async () => { + expect((await ctx.request('/api/auth/users/99999/sessions')).status).toBe(404) + expect((await ctx.request('/api/auth/users/abc/sessions')).status).toBe(400) + }) + + it('forbids a member from listing or revoking sessions', async () => { + const list = await ctx.request(`/api/auth/users/${memberId}/sessions`, { cookie: ctx.cookies.member }) + expect(list.status).toBe(403) + const row = await ctx.db.select().from(sessions).where(eq(sessions.userId, memberId)).get() + const revoke = await ctx.request(`/api/auth/sessions/${row!.id}`, { method: 'DELETE', cookie: ctx.cookies.member }) + expect(revoke.status).toBe(403) + }) + + it('ends the session on logout by its cookie token', async () => { + const res = await ctx.request('/api/auth/logout', { method: 'POST', cookie: ctx.cookies.member }) + expect(res.status).toBe(200) + const remaining = await ctx.db.select().from(sessions).where(eq(sessions.userId, memberId)).all() + expect(remaining).toHaveLength(0) + }) +}) + +describe('Switch user session', () => { + let ctx: AuthenticatedRouteTestContext + + beforeEach(async () => { + ctx = await setupAuthenticatedRouteTest({ + user: { email: 'first@test.com', role: 'member' }, + additionalUsers: [{ key: 'second', email: 'second@test.com', role: 'member' }], + mount: (app, { db }) => { + app.route('/api/auth', createAuthRoutes(db, db)) + }, + }) + }) + + afterEach(() => ctx.cleanup()) + + it('replaces the old session with a hashed session for the target user', async () => { + const second = await ctx.db.select().from(users).where(eq(users.email, 'second@test.com')).get() + const res = await ctx.request('/api/auth/switch', { + method: 'POST', + json: { userId: second!.id, password: 'password123' }, + }) + expect(res.status).toBe(200) + const newToken = cookieToken(res.headers.get('set-cookie') ?? '') + + expect((await ctx.request('/api/auth/me')).status).toBe(401) + const me = await ctx.request('/api/auth/me', { cookie: `session_id=${newToken}` }) + expect(me.status).toBe(200) + const row = await ctx.db.select().from(sessions).where(eq(sessions.tokenHash, hashSessionToken(newToken))).get() + expect(row?.userId).toBe(second!.id) + }) +}) diff --git a/packages/api/src/routes/__tests__/auth.test.ts b/packages/api/src/routes/__tests__/auth.test.ts index 02e98dd6..cffc1f76 100644 --- a/packages/api/src/routes/__tests__/auth.test.ts +++ b/packages/api/src/routes/__tests__/auth.test.ts @@ -8,7 +8,7 @@ import { setupTestDatabase, cleanupTestDatabase } from '../../__tests__/helpers/ import { users, sessions } from '../../db/schema' import { eq } from 'drizzle-orm' import bcrypt from 'bcryptjs' -import { BCRYPT_COST } from '../../lib/auth/auth-service' +import { BCRYPT_COST, hashSessionToken } from '../../lib/auth/auth-service' import type { Database } from '../../db/client' import { createTestUser, @@ -826,12 +826,14 @@ describe('Auth API', () => { // Create additional sessions for the user await testDb.insert(sessions).values({ id: 'session1', + tokenHash: hashSessionToken('session1-token'), userId: testUser.id, expiresAt: Math.floor(Date.now() / 1000) + 604800, }) await testDb.insert(sessions).values({ id: 'session2', + tokenHash: hashSessionToken('session2-token'), userId: testUser.id, expiresAt: Math.floor(Date.now() / 1000) + 604800, }) @@ -862,7 +864,7 @@ describe('Auth API', () => { .all() expect(remainingSessions.length).toBe(1) - expect(remainingSessions[0].id).toBe(currentSessionId) + expect(remainingSessions[0].tokenHash).toBe(hashSessionToken(currentSessionId)) }) }) diff --git a/packages/api/src/routes/__tests__/error-logs.test.ts b/packages/api/src/routes/__tests__/error-logs.test.ts index 601fb990..efe846c9 100644 --- a/packages/api/src/routes/__tests__/error-logs.test.ts +++ b/packages/api/src/routes/__tests__/error-logs.test.ts @@ -5,6 +5,8 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createErrorLogsRoutes } from '../error-logs' +import { ERROR_REPORT_LIMITS } from '@sampledb/contract' +import { errorLogs } from '../../db/schema' describe('Error Logs API', () => { let ctx: AuthenticatedRouteTestContext @@ -103,5 +105,73 @@ describe('Error Logs API', () => { }) expect(res.status).toBe(201) }) + + describe('size caps', () => { + const post = (body: string, headers: Record = {}) => + ctx.createRequestApp().request('/api/error-logs', { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body, + }) + + const countLogs = () => ctx.db.select().from(errorLogs).all().length + + it('accepts a report with every field at its cap', async () => { + const contextPadding = 'c'.repeat(ERROR_REPORT_LIMITS.contextJson - JSON.stringify({ pad: '' }).length) + const res = await post(JSON.stringify({ + message: 'm'.repeat(ERROR_REPORT_LIMITS.message), + stack: 's'.repeat(ERROR_REPORT_LIMITS.stack), + errorCode: 'e'.repeat(ERROR_REPORT_LIMITS.errorCode), + context: { pad: contextPadding }, + })) + expect(res.status).toBe(201) + expect(countLogs()).toBe(1) + }) + + it.each([ + ['message', { message: 'm'.repeat(ERROR_REPORT_LIMITS.message + 1) }], + ['stack', { message: 'x', stack: 's'.repeat(ERROR_REPORT_LIMITS.stack + 1) }], + ['errorCode', { message: 'x', errorCode: 'e'.repeat(ERROR_REPORT_LIMITS.errorCode + 1) }], + ['context', { message: 'x', context: { pad: 'c'.repeat(ERROR_REPORT_LIMITS.contextJson) } }], + ])('rejects a report whose %s is over its cap with 400 and writes no frontend log', async (_field, report) => { + const res = await post(JSON.stringify(report)) + expect(res.status).toBe(400) + const body = (await res.json()) as { errorCode?: string } + expect(body.errorCode).toBe('VALIDATION_ERROR') + expect(ctx.db.select().from(errorLogs).all().filter((l) => l.source === 'frontend')).toHaveLength(0) + }) + + it('rejects a body over the byte cap with 413 and writes no log', async () => { + const res = await post(JSON.stringify({ message: 'x', context: { pad: 'p'.repeat(ERROR_REPORT_LIMITS.bodyBytes) } })) + expect(res.status).toBe(413) + const body = (await res.json()) as { errorCode?: string } + expect(body.errorCode).toBe('PAYLOAD_TOO_LARGE') + expect(countLogs()).toBe(0) + }) + + it('rejects a streamed body over the byte cap (no Content-Length) with 413', async () => { + const chunk = new TextEncoder().encode('x'.repeat(16 * 1024)) + let sent = 0 + const stream = new ReadableStream({ + pull(controller) { + if (sent > ERROR_REPORT_LIMITS.bodyBytes) { + controller.close() + return + } + sent += chunk.length + controller.enqueue(chunk) + }, + }) + const res = await ctx.createRequestApp().request(new Request('http://localhost/api/error-logs', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: stream, + // @ts-expect-error - duplex is required by fetch for a streamed body but missing from the DOM types + duplex: 'half', + })) + expect(res.status).toBe(413) + expect(countLogs()).toBe(0) + }) + }) }) }) diff --git a/packages/api/src/routes/auth.ts b/packages/api/src/routes/auth.ts index 2e25ed10..794d593e 100644 --- a/packages/api/src/routes/auth.ts +++ b/packages/api/src/routes/auth.ts @@ -4,17 +4,18 @@ import { users, sessions } from '../db/schema' import { eq, and, isNull, gt, ne, or, sql } from 'drizzle-orm' import bcrypt from 'bcryptjs' import { z } from 'zod' -import { nanoid } from 'nanoid' import type { Database } from '../db/client' import { getPasswordRequirements, getSessionSettings } from '../lib/settings' import { createAuthMiddleware, createAdminMiddleware } from '../middleware/auth' import { rateLimit } from '../middleware/rate-limit' -import { handleRouteError } from '../lib/error-handler' +import { handleRouteError, NotFoundError, ValidationError } from '../lib/error-handler' import { toPublicUser, verifyLoginCredentials, verifyPasswordConstantTime, createUserSession, + deleteSessionByToken, + deleteUserSessions, registerApprovedUser, BCRYPT_COST, } from '../lib/auth/auth-service' @@ -82,7 +83,7 @@ auth.post('/login', rateLimit(10, 60 * 1000), async (c) => { const { emailOrUsername, password } = loginSchema.parse(body) const user = await verifyLoginCredentials(database, emailOrUsername, password) - const { sessionId, maxAgeSeconds } = await createUserSession(database, user.id) + const { sessionToken, maxAgeSeconds } = await createUserSession(database, user.id) // Update last login await database @@ -90,7 +91,7 @@ auth.post('/login', rateLimit(10, 60 * 1000), async (c) => { .set({ lastLogin: utcNow() }) .where(eq(users.id, user.id)) - setCookie(c, 'session_id', sessionId, { + setCookie(c, 'session_id', sessionToken, { httpOnly: true, secure: process.env.NODE_ENV === 'production', sameSite: 'lax', @@ -191,9 +192,9 @@ auth.post('/self-register', rateLimit(5, 60 * 1000), async (c) => { // Logout auth.post('/logout', async (c) => { - const sessionId = getCookie(c, 'session_id') - if (sessionId) { - await database.delete(sessions).where(eq(sessions.id, sessionId)) + const sessionToken = getCookie(c, 'session_id') + if (sessionToken) { + await deleteSessionByToken(database, sessionToken) } deleteCookie(c, 'session_id') return c.json({ message: 'Logged out' }) @@ -383,18 +384,8 @@ auth.patch('/me/password', authMiddleware, async (c) => { .where(eq(users.id, currentUser.id)) // Revoke all existing sessions except the current one for security - const sessionId = getCookie(c, 'session_id') - if (sessionId) { - await database - .delete(sessions) - .where(and( - eq(sessions.userId, currentUser.id), - ne(sessions.id, sessionId) - )) - } else { - // If no session cookie, revoke all sessions - await database.delete(sessions).where(eq(sessions.userId, currentUser.id)) - } + // (all of them when there is no session cookie) + await deleteUserSessions(database, currentUser.id, getCookie(c, 'session_id')) return c.json({ message: 'Password changed successfully' }) } catch (error) { @@ -476,23 +467,15 @@ auth.post('/switch', rateLimit(10, 60 * 1000), authMiddleware, async (c) => { if (!sessionSettings) { return c.json({ error: 'Session settings are not configured. Please run database initialization.' }, 500) } - const maxAgeSeconds = sessionSettings.maxAgeSeconds - + // Delete old session - const sessionId = getCookie(c, 'session_id') - if (sessionId) { - await database.delete(sessions).where(eq(sessions.id, sessionId)) + const oldSessionToken = getCookie(c, 'session_id') + if (oldSessionToken) { + await deleteSessionByToken(database, oldSessionToken) } // Create new session for target user - const newSessionId = nanoid() - const expiresAt = Math.floor(Date.now() / 1000) + maxAgeSeconds - - await database.insert(sessions).values({ - id: newSessionId, - userId: targetUser.id, - expiresAt, - }) + const { sessionToken, maxAgeSeconds } = await createUserSession(database, targetUser.id, dbForSettings) // Update last login await database @@ -500,7 +483,7 @@ auth.post('/switch', rateLimit(10, 60 * 1000), authMiddleware, async (c) => { .set({ lastLogin: utcNow() }) .where(eq(users.id, targetUser.id)) - setCookie(c, 'session_id', newSessionId, { + setCookie(c, 'session_id', sessionToken, { httpOnly: true, secure: process.env.NODE_ENV === 'production', sameSite: 'lax', @@ -724,7 +707,7 @@ auth.patch('/users/:id/password', adminMiddleware, async (c) => { .where(eq(users.id, id)) // Revoke all existing sessions for this user - await database.delete(sessions).where(eq(sessions.userId, id)) + await deleteUserSessions(database, id) return c.json({ message: 'Password reset successfully' }) } catch (error) { @@ -785,7 +768,7 @@ auth.delete('/users/:id', adminMiddleware, async (c) => { return c.json({ error: 'User not found' }, 404) } // Revoke all existing sessions for this user - await database.delete(sessions).where(eq(sessions.userId, id)) + await deleteUserSessions(database, id) return c.json({ message: 'User deleted successfully' }) } catch (error) { @@ -862,23 +845,24 @@ auth.post('/users/:id/restore', adminMiddleware, async (c) => { } }) -// Get active sessions for a user (admin only) +// Get active sessions for a user (admin only). Each session is listed by its public id, which +// revokes it through DELETE /sessions/:id but cannot be used as a session cookie. auth.get('/users/:id/sessions', adminMiddleware, async (c) => { try { const id = parseInt(requireParam(c, 'id')) if (isNaN(id)) { - return c.json({ error: 'Invalid user ID' }, 400) + throw new ValidationError('Invalid user ID') } // Check if user exists const user = await database - .select() + .select({ id: users.id }) .from(users) .where(eq(users.id, id)) .get() if (!user) { - return c.json({ error: 'User not found' }, 404) + throw new NotFoundError('User', id) } const now = Math.floor(Date.now() / 1000) @@ -895,27 +879,27 @@ auth.get('/users/:id/sessions', adminMiddleware, async (c) => { return c.json({ sessions: activeSessions }) } catch (error) { - return c.json({ error: 'Internal server error' }, 500) + return handleRouteError(error, c) } }) -// Revoke a session (admin only) +// Revoke a session by its public id (admin only) auth.delete('/sessions/:id', adminMiddleware, async (c) => { try { - const sessionId = requireParam(c, 'id') + const publicId = requireParam(c, 'id') const deleted = await database .delete(sessions) - .where(eq(sessions.id, sessionId)) - .returning() + .where(eq(sessions.id, publicId)) + .returning({ id: sessions.id }) if (deleted.length === 0) { - return c.json({ error: 'Session not found' }, 404) + throw new NotFoundError('Session') } return c.json({ message: 'Session revoked successfully' }) } catch (error) { - return c.json({ error: 'Internal server error' }, 500) + return handleRouteError(error, c) } }) diff --git a/packages/api/src/routes/error-logs.ts b/packages/api/src/routes/error-logs.ts index 9ebdfd8f..f4eeb849 100644 --- a/packages/api/src/routes/error-logs.ts +++ b/packages/api/src/routes/error-logs.ts @@ -1,9 +1,11 @@ import { Hono } from 'hono' +import { bodyLimit } from 'hono/body-limit' import { z } from 'zod' +import { ERROR_REPORT_LIMITS, errorReportSchema } from '@sampledb/contract' import type { Database } from '../db/client' import { errorLogs } from '../db/schema' import { logFrontendError, cleanupOldErrorLogs, type ErrorLogContext } from '../lib/error-logger' -import { handleRouteError } from '../lib/error-handler' +import { handleRouteError, PayloadTooLargeError } from '../lib/error-handler' import { eq, and, desc, sql, or } from 'drizzle-orm' import { createAdminMiddleware, createAuthMiddleware, createOptionalAuthMiddleware } from '../middleware/auth' import { rateLimit } from '../middleware/rate-limit' @@ -11,14 +13,9 @@ import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' import { likeContains } from '../lib/sql-like' -// Schema for frontend error submission -const frontendErrorSchema = z.object({ - message: z.string(), - stack: z.string().optional(), - errorCode: z.string().optional(), - level: z.enum(['error', 'warning', 'info']).default('error'), - context: z.record(z.string(), z.unknown()).optional(), -}) +/** Header values copied into an error log row are cut to these lengths. */ +const MAX_LOGGED_URL_LENGTH = 2_000 +const MAX_LOGGED_USER_AGENT_LENGTH = 500 // Schema for error log query parameters const errorLogQuerySchema = z.object({ @@ -64,17 +61,24 @@ export function createErrorLogsRoutes(database: Database): Hono { // POST /api/error-logs - Accept frontend error reports // Use optional auth so errors can be logged even if session expired - // Rate limited to prevent error-log flooding (60 requests per minute per IP) - errorLogsRoutes.post('/', rateLimit(60, 60 * 1000), optionalAuthMiddleware, async (c) => { + // Rate limited to prevent error-log flooding (60 requests per minute per IP), and capped in + // size (ERROR_REPORT_LIMITS): 413 for an oversized body, 400 for an oversized field. + const errorReportBodyLimit = bodyLimit({ + maxSize: ERROR_REPORT_LIMITS.bodyBytes, + onError: () => { + throw new PayloadTooLargeError(`Error report body is over ${ERROR_REPORT_LIMITS.bodyBytes} bytes`) + }, + }) + errorLogsRoutes.post('/', rateLimit(60, 60 * 1000), errorReportBodyLimit, optionalAuthMiddleware, async (c) => { try { const body = await c.req.json() - const errorData = frontendErrorSchema.parse(body) + const errorData = errorReportSchema.parse(body) // Extract context from request // User may be undefined if not authenticated (e.g., session expired) const user = c.get('user') - const url = c.req.header('referer') || c.req.url - const userAgent = c.req.header('user-agent') + const url = (c.req.header('referer') || c.req.url).slice(0, MAX_LOGGED_URL_LENGTH) + const userAgent = c.req.header('user-agent')?.slice(0, MAX_LOGGED_USER_AGENT_LENGTH) const errorContext: ErrorLogContext = { userId: user?.id, diff --git a/packages/contract/src/__tests__/error-report.test.ts b/packages/contract/src/__tests__/error-report.test.ts new file mode 100644 index 00000000..90b6aa3c --- /dev/null +++ b/packages/contract/src/__tests__/error-report.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect } from 'bun:test' +import { ERROR_REPORT_LIMITS, errorReportSchema, fitErrorReportToLimits } from '../error-report' + +describe('errorReportSchema', () => { + it('accepts a report at every cap and rejects one over any cap', () => { + const atCap = { + message: 'm'.repeat(ERROR_REPORT_LIMITS.message), + stack: 's'.repeat(ERROR_REPORT_LIMITS.stack), + errorCode: 'e'.repeat(ERROR_REPORT_LIMITS.errorCode), + } + expect(errorReportSchema.safeParse(atCap).success).toBe(true) + expect(errorReportSchema.safeParse({ ...atCap, message: atCap.message + 'm' }).success).toBe(false) + expect(errorReportSchema.safeParse({ ...atCap, stack: atCap.stack + 's' }).success).toBe(false) + expect(errorReportSchema.safeParse({ ...atCap, errorCode: atCap.errorCode + 'e' }).success).toBe(false) + expect( + errorReportSchema.safeParse({ message: 'x', context: { pad: 'c'.repeat(ERROR_REPORT_LIMITS.contextJson) } }).success + ).toBe(false) + }) +}) + +describe('fitErrorReportToLimits', () => { + it('leaves a report within the caps unchanged', () => { + const report = { message: 'm', stack: 's', level: 'error' as const, context: { a: 1 } } + expect(fitErrorReportToLimits(report)).toEqual(report) + }) + + it('cuts long strings and replaces an oversized context with the kept keys', () => { + const fitted = fitErrorReportToLimits( + { + message: 'm'.repeat(ERROR_REPORT_LIMITS.message + 10), + stack: 's'.repeat(ERROR_REPORT_LIMITS.stack + 10), + errorCode: 'e'.repeat(ERROR_REPORT_LIMITS.errorCode + 10), + context: { requestId: 'req_1', count: 3, blob: 'b'.repeat(ERROR_REPORT_LIMITS.contextJson) }, + }, + ['requestId', 'count', 'missing'] + ) + expect(errorReportSchema.safeParse(fitted).success).toBe(true) + expect(fitted.message.endsWith('…[truncated]')).toBe(true) + expect(fitted.context).toEqual({ contextTruncated: true, requestId: 'req_1' }) + }) +}) diff --git a/packages/contract/src/error-report.ts b/packages/contract/src/error-report.ts new file mode 100644 index 00000000..6115b749 --- /dev/null +++ b/packages/contract/src/error-report.ts @@ -0,0 +1,66 @@ +import { z } from 'zod' + +/** + * Size caps on a frontend error report (POST /api/error-logs). The endpoint accepts reports + * without login, so the caps bound how much one request can write to `error_logs`. + */ +export const ERROR_REPORT_LIMITS = { + /** Whole request body, in bytes; the API rejects a larger body with 413. */ + bodyBytes: 64 * 1024, + message: 2_000, + stack: 16_000, + errorCode: 200, + /** `context` serialized as JSON, in characters. */ + contextJson: 16_000, +} as const + +const ERROR_REPORT_LEVELS = ['error', 'warning', 'info'] as const + +function jsonLength(value: unknown): number { + return JSON.stringify(value)?.length ?? 0 +} + +export const errorReportSchema = z.object({ + message: z.string().max(ERROR_REPORT_LIMITS.message), + stack: z.string().max(ERROR_REPORT_LIMITS.stack).optional(), + errorCode: z.string().max(ERROR_REPORT_LIMITS.errorCode).optional(), + level: z.enum(ERROR_REPORT_LEVELS).default('error'), + context: z + .record(z.string(), z.unknown()) + .refine((context) => jsonLength(context) <= ERROR_REPORT_LIMITS.contextJson, { + message: `context must serialize to at most ${ERROR_REPORT_LIMITS.contextJson} characters of JSON`, + }) + .optional(), +}) + +export type ErrorReport = z.input + +const TRUNCATION_MARK = '…[truncated]' + +function truncate(value: string, max: number): string { + return value.length <= max ? value : value.slice(0, max - TRUNCATION_MARK.length) + TRUNCATION_MARK +} + +/** + * Shorten a report so it passes `errorReportSchema`'s caps: long strings are cut, and a context + * over its cap is replaced by the `keepContextKeys` entries that are strings (each cut to 1,000 + * characters) plus `contextTruncated: true`. + */ +export function fitErrorReportToLimits(report: ErrorReport, keepContextKeys: readonly string[] = []): ErrorReport { + let context = report.context + if (context && jsonLength(context) > ERROR_REPORT_LIMITS.contextJson) { + const kept: Record = { contextTruncated: true } + for (const key of keepContextKeys) { + const value = context[key] + if (typeof value === 'string') kept[key] = truncate(value, 1_000) + } + context = kept + } + return { + ...report, + message: truncate(report.message, ERROR_REPORT_LIMITS.message), + ...(report.stack !== undefined && { stack: truncate(report.stack, ERROR_REPORT_LIMITS.stack) }), + ...(report.errorCode !== undefined && { errorCode: truncate(report.errorCode, ERROR_REPORT_LIMITS.errorCode) }), + ...(context !== undefined && { context }), + } +} diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1a9f103e..e123d667 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -81,3 +81,9 @@ export { PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type PaperInboundWriteFields, } from './paper-container-inbound' +export { + ERROR_REPORT_LIMITS, + errorReportSchema, + fitErrorReportToLimits, + type ErrorReport, +} from './error-report' diff --git a/packages/docs/src/content/docs/guides/troubleshooting/release-notes.md b/packages/docs/src/content/docs/guides/troubleshooting/release-notes.md index 063b4c86..8b91e2b0 100644 --- a/packages/docs/src/content/docs/guides/troubleshooting/release-notes.md +++ b/packages/docs/src/content/docs/guides/troubleshooting/release-notes.md @@ -32,6 +32,16 @@ SampleDB **container export** CSV files (Multi-Study Export, Study export modal, - **Collection table snapshot export** on collection detail pages uses the same CSV download conventions; cell content was not wrapped in Excel formulas. - Import templates and **Export filter file** uploads are separate from download format; see [CSV file guidelines](/docs/guides/troubleshooting/csv-guidelines/). +## Sign-in sessions and error reporting (security hardening) + +**Everyone signs in again once.** After this upgrade, every existing session ends and each user sees the sign-in page on their next request. SampleDB now stores only a hash of each session's cookie, and sessions saved by the old version can't be converted, so the upgrade removes them. + +**What changed** + +- In **Admin > Users > View sessions**, the ID shown for each session is a separate public ID. Use it to revoke the session; it can't be used to sign in as the user. +- Error reports that the browser sends to **Error Logs** have size limits: 2,000 characters for the message, 16,000 for the stack trace, 200 for the error code, 16,000 characters of JSON for the context, and 64 KB for the whole request. The API rejects a larger report. The SampleDB web app shortens its own reports to fit. +- When the server fails with an unexpected error in production, the response says only `Internal server error`. The full error, with its stack trace, still goes to the server log and to **Error Logs**. + ## Related documentation - [Multi-study export](/docs/guides/bulk-operations/export/): CSV vs XLSX guidance for multi-study container export diff --git a/packages/web/src/lib/__tests__/error-logger.test.ts b/packages/web/src/lib/__tests__/error-logger.test.ts index bdd90159..5d0b88dd 100644 --- a/packages/web/src/lib/__tests__/error-logger.test.ts +++ b/packages/web/src/lib/__tests__/error-logger.test.ts @@ -10,6 +10,7 @@ vi.mock('../api/client', () => ({ getLastResponseRequestId: getLastResponseRequestIdMock, })) +import { ERROR_REPORT_LIMITS, errorReportSchema } from '@sampledb/contract' import { logErrorFromMessage } from '../error-logger' describe('error-logger', () => { @@ -62,4 +63,24 @@ describe('error-logger', () => { logErrorFromMessage('msg', 'warning', { key: 'value' }), ).not.toThrow() }) + + it('cuts an oversized report to the API size caps before posting it', async () => { + logErrorFromMessage('m'.repeat(ERROR_REPORT_LIMITS.message * 2), 'error', { + requestId: 'req_big', + blob: 'b'.repeat(ERROR_REPORT_LIMITS.contextJson * 2), + }) + await vi.runAllTimersAsync() + + // The module-level queue may still hold reports from earlier tests; pick this one out. + const sent = postMock.mock.calls + .map((call) => call[1] as { message: string; context: Record }) + .find((report) => report.context.requestId === 'req_big') + if (!sent) throw new Error('report was not posted') + expect(errorReportSchema.safeParse(sent).success).toBe(true) + expect(sent.message).toHaveLength(ERROR_REPORT_LIMITS.message) + expect(sent.context).toEqual( + expect.objectContaining({ contextTruncated: true, requestId: 'req_big', url: window.location.href }), + ) + expect(sent.context).not.toHaveProperty('blob') + }) }) diff --git a/packages/web/src/lib/api/admin.ts b/packages/web/src/lib/api/admin.ts index 19b57db1..0fa4ecfd 100644 --- a/packages/web/src/lib/api/admin.ts +++ b/packages/web/src/lib/api/admin.ts @@ -55,8 +55,9 @@ export const adminApi = { api.patch<{ message: string }>(`/auth/users/${id}/password`, { password }), getUserSessions: (userId: number) => api.get<{ sessions: UserSession[] }>(`/auth/users/${userId}/sessions`), - revokeSession: (sessionId: string) => - api.delete<{ message: string }>(`/auth/sessions/${sessionId}`), + /** Revoke a session by its public id (`UserSession.id`). */ + revokeSession: (publicSessionId: string) => + api.delete<{ message: string }>(`/auth/sessions/${encodeURIComponent(publicSessionId)}`), approveUser: (id: number) => api.patch<{ user: User }>(`/auth/users/${id}/approve`), getSystemStats: () => diff --git a/packages/web/src/lib/api/auth.ts b/packages/web/src/lib/api/auth.ts index 5cbef387..f68ad747 100644 --- a/packages/web/src/lib/api/auth.ts +++ b/packages/web/src/lib/api/auth.ts @@ -12,6 +12,7 @@ export interface User { } export interface UserSession { + /** Public session id for listing and revoking; it is not the session cookie and cannot log in. */ id: string expiresAt: number } diff --git a/packages/web/src/lib/error-logger.ts b/packages/web/src/lib/error-logger.ts index 9df3fb33..db17bc27 100644 --- a/packages/web/src/lib/error-logger.ts +++ b/packages/web/src/lib/error-logger.ts @@ -1,3 +1,4 @@ +import { fitErrorReportToLimits } from '@sampledb/contract' import { axiosApi, getLastResponseRequestId } from './api/client' export interface FrontendError { @@ -8,6 +9,9 @@ export interface FrontendError { context?: Record } +/** Context entries a report keeps when its context is over the API's size cap. */ +const REPORT_CONTEXT_KEYS_KEPT_WHEN_TRUNCATED = ['requestId', 'userAgent', 'url', 'timestamp'] as const + // Error queue to batch errors and avoid overwhelming the server const errorQueue: FrontendError[] = [] const MAX_QUEUE_SIZE = 10 @@ -23,19 +27,24 @@ async function sendError(error: FrontendError): Promise { getLastResponseRequestId() try { - await axiosApi.post('/error-logs', { - message: error.message, - stack: error.stack, - errorCode: error.errorCode, - level: error.level, - context: { - ...error.context, - ...(requestId ? { requestId } : {}), - userAgent: navigator.userAgent, - url: window.location.href, - timestamp: new Date().toISOString(), + // The API rejects a report over its size caps, so cut it to fit rather than lose it. + const report = fitErrorReportToLimits( + { + message: error.message, + stack: error.stack, + errorCode: error.errorCode, + level: error.level, + context: { + ...error.context, + ...(requestId ? { requestId } : {}), + userAgent: navigator.userAgent, + url: window.location.href, + timestamp: new Date().toISOString(), + }, }, - }) + REPORT_CONTEXT_KEYS_KEPT_WHEN_TRUNCATED + ) + await axiosApi.post('/error-logs', report) } catch (err) { // If logging fails, fall back to console console.error('[ERROR_LOGGER] Failed to send error to backend:', err) diff --git a/packages/web/src/pages/__tests__/AdminUsers.test.tsx b/packages/web/src/pages/__tests__/AdminUsers.test.tsx index 762fdc59..3a0c0b2c 100644 --- a/packages/web/src/pages/__tests__/AdminUsers.test.tsx +++ b/packages/web/src/pages/__tests__/AdminUsers.test.tsx @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { render, screen, waitFor } from '../../__tests__/helpers/render' import AdminUsers from '../AdminUsers' import { adminApi } from '../../lib/api/admin' +import userEvent from '@testing-library/user-event' vi.mock('../../lib/api/admin', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -14,6 +15,7 @@ vi.mock('../../lib/api/admin', async () => { restoreUser: vi.fn().mockResolvedValue(undefined), resetPassword: vi.fn().mockResolvedValue(undefined), getUserSessions: vi.fn().mockResolvedValue({ sessions: [] }), + revokeSession: vi.fn().mockResolvedValue({ message: 'Session revoked successfully' }), } }) }) @@ -50,4 +52,23 @@ describe('AdminUsers', () => { expect(screen.getByText('Admin User')).toBeInTheDocument() }, { timeout: 3000 }) }) + + it('lists a user\'s sessions by public id and revokes one by that id', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.getUsers).mockResolvedValue({ + users: [{ id: 7, name: 'Member User', email: 'member@test.com', role: 'member', approvedAt: '2026-01-01' }], + }) + vi.mocked(adminApi.getUserSessions) + .mockResolvedValueOnce({ sessions: [{ id: 'pub_abc', expiresAt: 2_000_000_000 }] }) + .mockResolvedValueOnce({ sessions: [] }) + await render() + + await user.click(await screen.findByRole('button', { name: /view sessions/i })) + expect(await screen.findByText('pub_abc')).toBeInTheDocument() + expect(adminApi.getUserSessions).toHaveBeenCalledWith(7) + + await user.click(screen.getByRole('button', { name: /revoke/i })) + await waitFor(() => expect(adminApi.revokeSession).toHaveBeenCalledWith('pub_abc')) + expect(await screen.findByText(/no active sessions/i)).toBeInTheDocument() + }) }) From 2605018bb48840309737a1bb586f18117dc70695 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:35:05 -0700 Subject: [PATCH 08/54] fix: store every Specimen collection date as YYYY-MM-DD (#104) - Migration 008 rewrites legacy M/D/YY and M/D/YYYY collection dates (2-digit year read as 20YY), trims whitespace, turns blank values into NULL, and aborts with a message if any other value remains. - List, statistics and Container selection (export) date filters reject a malformed bound with a 400 instead of comparing it as a string. - Derivation refuses to copy a non-ISO parent date; the derivation CSV normalizes parent_collection_date and rejects a value that isn't a date. - The integrity report gains a "Collection date not ISO" check. Co-Authored-By: Claude Opus 5.5 --- packages/api/initial_schema.sql | 2 +- packages/api/src/db/__tests__/client.test.ts | 1 + .../src/db/__tests__/schema-evolution.test.ts | 63 ++++++++++++++++++ .../db/migrations/008_collection_date_iso.sql | 52 +++++++++++++++ packages/api/src/db/schema-evolution.ts | 2 +- .../api/src/lib/__tests__/derivations.test.ts | 38 +++++++++++ .../api/src/lib/container-selection/index.ts | 40 +++++++++++- packages/api/src/lib/data-audit.ts | 25 +++++++ packages/api/src/lib/date-filters.ts | 45 +++++++++++++ packages/api/src/lib/derivations.ts | 8 +++ .../parent-container-resolver.test.ts | 32 +++++++++ .../derivations/parent-container-resolver.ts | 27 +++++--- .../api/src/lib/specimens/specimen-read.ts | 21 +++--- .../api/src/lib/statistics/parse-filters.ts | 14 ++-- .../src/routes/__tests__/data-audit.test.ts | 20 ++++++ .../api/src/routes/__tests__/export.test.ts | 39 +++++++++++ .../src/routes/__tests__/specimens.test.ts | 65 +++++++++++++++++++ .../src/routes/__tests__/statistics.test.ts | 26 +++++++- .../docs/guides/advanced/data-integrity.md | 1 + packages/web/src/lib/api/admin.ts | 6 ++ .../src/pages/AdminDataIntegrityOverview.tsx | 3 +- .../src/pages/AdminDataIntegrityReport.tsx | 29 +++++++++ ...dminDataIntegrityEmptyCollections.test.tsx | 1 + .../AdminDataIntegrityOverview.test.tsx | 15 +++++ .../AdminDataIntegrityReport.test.tsx | 12 ++++ 25 files changed, 560 insertions(+), 27 deletions(-) create mode 100644 packages/api/src/db/migrations/008_collection_date_iso.sql create mode 100644 packages/api/src/lib/date-filters.ts diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index ce2b56e2..f07e62c7 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -284,7 +284,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (7);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (8);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/client.test.ts b/packages/api/src/db/__tests__/client.test.ts index c08ee650..ba181e14 100644 --- a/packages/api/src/db/__tests__/client.test.ts +++ b/packages/api/src/db/__tests__/client.test.ts @@ -34,6 +34,7 @@ describe('Database Client', () => { db.exec(`CREATE TABLE static_well (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, position TEXT)`) db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL)`) db.exec(`CREATE TABLE sheet (id INTEGER PRIMARY KEY, name TEXT NOT NULL, box_id INTEGER, bag_id INTEGER)`) + db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, specimen_type_id INTEGER NOT NULL, collection_date DATE)`) // Intentionally omit paper — migration 003 preflight creates legacy stub // Intentionally do NOT create error_logs const before = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='error_logs'").get() diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index 65caa5ae..d2725707 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -42,6 +42,12 @@ function createTablesForLaterMigrations(sqlite: Database): void { box_id INTEGER, bag_id INTEGER )`) + sqlite.exec(`CREATE TABLE IF NOT EXISTS specimen ( + id INTEGER PRIMARY KEY, + study_subject_id INTEGER, + specimen_type_id INTEGER NOT NULL, + collection_date DATE + )`) } describe('schema evolution', () => { @@ -293,6 +299,61 @@ describe('schema evolution', () => { sqlite.close() }) + it('migration 008 rewrites M/D/YY and M/D/YYYY collection dates as YYYY-MM-DD', () => { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (7)') + createTablesForLaterMigrations(sqlite) + const values: Array<[number, string | null]> = [ + [1, '5/7/26'], + [2, '9/14/20'], + [3, '12/24/2024'], + [4, '07/04/2024'], + [5, '2023-08-21'], + [6, null], + [7, ' '], + [8, ' 3/17/22 '], + [9, '1970-01-01'], + ] + const insert = sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (?, 1, ?)') + for (const [id, date] of values) insert.run(id, date) + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + const rows = sqlite.prepare('SELECT collection_date AS date FROM specimen ORDER BY id').all() as Array<{ date: string | null }> + expect(rows.map((r) => r.date)).toEqual([ + '2026-05-07', + '2020-09-14', + '2024-12-24', + '2024-07-04', + '2023-08-21', + null, + null, + '2022-03-17', + '1970-01-01', + ]) + sqlite.close() + }) + + it.each(['13/2/2024', '2/30/2024', '2024-02-30', '5/7', '7 Apr 2024', '2024/05/07', '5/7/124'])( + 'migration 008 aborts on a collection date it cannot read (%s) and changes nothing', + (bad) => { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (7)') + createTablesForLaterMigrations(sqlite) + sqlite.exec(`INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (1, 1, '5/7/26')`) + sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (2, 1, ?)').run(bad) + + expect(() => evolveOperationalSchema(sqlite)).toThrow(/Cannot convert collection dates to YYYY-MM-DD/) + expect(getRecordedSchemaVersion(sqlite)).toBe(7) + const rows = sqlite.prepare('SELECT collection_date AS date FROM specimen ORDER BY id').all() as Array<{ date: string }> + expect(rows.map((r) => r.date)).toEqual(['5/7/26', bad]) + sqlite.close() + }, + ) + /** The CREATE TABLE text SQLite stored for qpcr_well_result. */ function qpcrWellResultDdl(sqlite: Database): string { const row = sqlite.prepare("SELECT sql FROM sqlite_master WHERE type='table' AND name='qpcr_well_result'").get() as @@ -311,6 +372,7 @@ describe('schema evolution', () => { const migrated = new Database(testDbPath) migrated.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') migrated.exec('INSERT INTO schema_version (version) VALUES (6)') + createTablesForLaterMigrations(migrated) evolveOperationalSchema(migrated) expect(getRecordedSchemaVersion(migrated)).toBe(CURRENT_SCHEMA_VERSION) expect(qpcrWellResultDdl(migrated)).toContain('CONSTRAINT qpcr_well_result_run_well_target_unique UNIQUE') @@ -329,6 +391,7 @@ describe('schema evolution', () => { UNIQUE (qpcr_run_id, well_position, target_name) )` sqlite.exec(existing) + createTablesForLaterMigrations(sqlite) evolveOperationalSchema(sqlite) diff --git a/packages/api/src/db/migrations/008_collection_date_iso.sql b/packages/api/src/db/migrations/008_collection_date_iso.sql new file mode 100644 index 00000000..a8e9e2e9 --- /dev/null +++ b/packages/api/src/db/migrations/008_collection_date_iso.sql @@ -0,0 +1,52 @@ +-- Migration 008: every Specimen collection date is YYYY-MM-DD (issue #104). +-- Rewrites legacy M/D/YY and M/D/YYYY values (US month first; a 2-digit year is 20YY) and +-- surrounding whitespace, and turns blank values into NULL. Abort if any other value remains, +-- so it can be reviewed by hand. +UPDATE specimen SET collection_date = NULL +WHERE typeof(collection_date) = 'text' AND trim(collection_date) = '';--> statement-breakpoint +CREATE TEMP TABLE __migration_008_dates AS +WITH legacy AS ( + SELECT id, trim(collection_date) AS v + FROM specimen + WHERE collection_date IS NOT NULL + AND NOT ( + typeof(collection_date) = 'text' + AND collection_date GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' + AND date(collection_date) IS collection_date + ) +), +parts AS ( + SELECT id, v, substr(v, 1, instr(v, '/') - 1) AS m, substr(v, instr(v, '/') + 1) AS rest FROM legacy +), +fields AS ( + SELECT id, v, m, substr(rest, 1, instr(rest, '/') - 1) AS d, substr(rest, instr(rest, '/') + 1) AS y FROM parts +), +candidates AS ( + SELECT id, CASE + WHEN v GLOB '[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]' THEN v + WHEN (m GLOB '[0-9]' OR m GLOB '[0-9][0-9]') + AND (d GLOB '[0-9]' OR d GLOB '[0-9][0-9]') + AND (y GLOB '[0-9][0-9]' OR y GLOB '[0-9][0-9][0-9][0-9]') + THEN printf( + '%04d-%02d-%02d', + CASE WHEN length(y) = 2 THEN 2000 + CAST(y AS INTEGER) ELSE CAST(y AS INTEGER) END, + CAST(m AS INTEGER), + CAST(d AS INTEGER) + ) + END AS iso + FROM fields +) +SELECT id, CASE WHEN date(iso) IS iso THEN iso END AS iso FROM candidates;--> statement-breakpoint +CREATE TEMP TABLE IF NOT EXISTS __migration_008_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint +CREATE TRIGGER __migration_008_guard_tr BEFORE INSERT ON __migration_008_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot convert collection dates to YYYY-MM-DD: a specimen.collection_date is not YYYY-MM-DD, M/D/YY or M/D/YYYY, or is not a real calendar date. Fix it by hand first. SELECT id, collection_date FROM specimen WHERE date(collection_date) IS NOT collection_date lists every value that is not YYYY-MM-DD.') + WHERE EXISTS (SELECT 1 FROM __migration_008_dates WHERE iso IS NULL); +END;--> statement-breakpoint +INSERT INTO __migration_008_guard DEFAULT VALUES;--> statement-breakpoint +DROP TRIGGER __migration_008_guard_tr;--> statement-breakpoint +DROP TABLE __migration_008_guard;--> statement-breakpoint +UPDATE specimen +SET collection_date = (SELECT iso FROM __migration_008_dates WHERE __migration_008_dates.id = specimen.id) +WHERE id IN (SELECT id FROM __migration_008_dates);--> statement-breakpoint +DROP TABLE __migration_008_dates; diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index b96368bd..3598e78a 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -7,7 +7,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 7 +export const CURRENT_SCHEMA_VERSION = 8 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/lib/__tests__/derivations.test.ts b/packages/api/src/lib/__tests__/derivations.test.ts index 751d8d1c..797aeefc 100644 --- a/packages/api/src/lib/__tests__/derivations.test.ts +++ b/packages/api/src/lib/__tests__/derivations.test.ts @@ -198,6 +198,44 @@ describe('derivations', () => { expect(result.specimen.collectionDate).toBeNull() }) + it('refuses to copy a parent collection date that is not YYYY-MM-DD to the child', async () => { + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await setContainerDefaults(testDb, { + micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + cryovial_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + paper: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + static_well: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + }) + const bloodType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const dnaType = await createTestSpecimenType(testDb, { name: 'DNA' }) + const now = utcNow() + await testDb.insert(specimenTypeContainerType).values({ specimenTypeId: dnaType.id, containerType: 'micronix_tube', created: now }) + await testDb.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const parentSpecimen = await createTestSpecimen(testDb, bloodType.id, { collectionDate: '5/7/24' }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: location.id }) + const [parentContainer] = await testDb + .insert(storageContainer) + .values({ specimenId: parentSpecimen.id, unitId: unit.id, totalQuantity: 1, remainingQuantity: 1, created: now, lastUpdated: now }) + .returning() + await testDb.insert(micronixTube).values({ id: parentContainer!.id, collectionId: plate.id, barcode: 'MT-LEGACY', position: 'A01' }) + + await expect( + createDerivation(testDb, { + parentContainerId: parentContainer!.id, + derivationType: 'extraction', + specimenTypeName: 'DNA', + container: { + containerType: 'micronix_tube', + barcode: 'MT-LEGACY-CHILD', + collection: { type: 'micronix_plate', id: plate.id, position: 'A02' }, + }, + }), + ).rejects.toThrow(/'5\/7\/24', which is not YYYY-MM-DD/) + expect(await testDb.select().from(specimen).where(eq(specimen.specimenTypeId, dnaType.id))).toEqual([]) + }) + it('rolls back the derived specimen when child container creation fails', async () => { const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) await setContainerDefaults(testDb, { diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 96327127..7be0fbcd 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -7,6 +7,7 @@ import { and, eq, getTableColumns, inArray, type SQL } from 'drizzle-orm' import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject } from '../../db/schema' import { CONTAINER_SUBTYPES, type ContainerSubtype } from '../container-placement' +import { collectionDateFilter, timestampFilter } from '../date-filters' import { containerFilterConditions, containerTypeSql, @@ -17,6 +18,7 @@ import { import { micronixBarcodeScopes, subjectListScopes, type SelectionScope } from './scopes' import type { ContainerSelection, + ContainerSelectionFilters, ContainerSelectionResult, ContainerSelectionTypes, SelectedContainer, @@ -81,7 +83,8 @@ async function selectionScopes( return { scopes: micronixBarcodeScopes(database, selection.barcodes), containerConditions: [] } } - const filters = selection.filters ?? {} + const filters = checkedDateFilters(selection.filters ?? {}) + if (selection.kind === 'subjects') checkSubjectDates(selection) const collectionDate = dateRangeConditions(specimen.collectionDate, filters.collectionDate) let scopes: SelectionScope[] switch (selection.kind) { @@ -110,6 +113,41 @@ async function selectionScopes( } } +/** + * The filters with each date bound checked: a malformed bound throws a `ValidationError` (400) + * rather than matching everything or nothing as a string comparison would (issue #104). + */ +function checkedDateFilters(filters: ContainerSelectionFilters): ContainerSelectionFilters { + return { + ...filters, + collectionDate: { + from: collectionDateFilter('collection date from', filters.collectionDate?.from), + to: collectionDateFilter('collection date to', filters.collectionDate?.to), + }, + specimenCreated: { + from: timestampFilter('Specimen created from', filters.specimenCreated?.from), + to: timestampFilter('Specimen created to', filters.specimenCreated?.to), + }, + containerCreated: { + from: timestampFilter('Container created from', filters.containerCreated?.from), + to: timestampFilter('Container created to', filters.containerCreated?.to), + }, + } +} + +/** Check every per-Subject collection date of a subject-list selection; see `checkedDateFilters`. */ +function checkSubjectDates(selection: Extract): void { + for (const entry of selection.subjects) { + for (const date of entry.dates ?? []) { + if ('exact' in date) collectionDateFilter('collection date', date.exact) + else { + collectionDateFilter('collection date from', date.from) + collectionDateFilter('collection date to', date.to) + } + } + } +} + /** * Run each scope (one per chunk of a listed selection) and collect its Specimens and the * Containers that also pass the Container-level conditions. Scopes may overlap, so rows are diff --git a/packages/api/src/lib/data-audit.ts b/packages/api/src/lib/data-audit.ts index 6311008c..d43a001b 100644 --- a/packages/api/src/lib/data-audit.ts +++ b/packages/api/src/lib/data-audit.ts @@ -20,6 +20,7 @@ import { tag, } from '../db/schema' import { eq, inArray, sql } from 'drizzle-orm' +import { isIsoCalendarDate } from './date-filters' export type CollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' @@ -341,6 +342,11 @@ export interface ContainerWithNoGridPositionItem { collectionId: number } +export interface SpecimenWithNonIsoCollectionDateItem { + id: number + collectionDate: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -354,6 +360,7 @@ export interface IntegrityReport { duplicateBarcodes: DuplicateBarcodeItem[] locationPathInconsistencies: LocationPathInconsistencyItem[] containersWithNoGridPosition: ContainerWithNoGridPositionItem[] + specimensWithNonIsoCollectionDate: SpecimenWithNonIsoCollectionDateItem[] } /** Collections (plates, boxes, bags) whose locationId does not exist in location. */ @@ -520,6 +527,21 @@ export async function listContainersWithNoGridPosition(db: Database): Promise { + const rows = await db + .select({ id: specimen.id, collectionDate: specimen.collectionDate }) + .from(specimen) + .where(sql`${specimen.collectionDate} IS NOT NULL AND date(${specimen.collectionDate}) IS NOT ${specimen.collectionDate}`) + .orderBy(specimen.id) + return rows + .map((row) => ({ id: row.id, collectionDate: String(row.collectionDate) })) + .filter((row) => !isIsoCalendarDate(row.collectionDate)) +} + /** Build full integrity report (all checks). */ export async function getIntegrityReport(db: Database): Promise { const [ @@ -535,6 +557,7 @@ export async function getIntegrityReport(db: Database): Promise duplicateBarcodes, locationPathInconsistencies, containersWithNoGridPosition, + specimensWithNonIsoCollectionDate, ] = await Promise.all([ listEmptyCollections(db), listCollectionsWithMissingLocation(db), @@ -548,6 +571,7 @@ export async function getIntegrityReport(db: Database): Promise listDuplicateBarcodes(db), listLocationPathInconsistencies(db), listContainersWithNoGridPosition(db), + listSpecimensWithNonIsoCollectionDate(db), ]) return { emptyCollections, @@ -562,5 +586,6 @@ export async function getIntegrityReport(db: Database): Promise duplicateBarcodes, locationPathInconsistencies, containersWithNoGridPosition, + specimensWithNonIsoCollectionDate, } } diff --git a/packages/api/src/lib/date-filters.ts b/packages/api/src/lib/date-filters.ts new file mode 100644 index 00000000..244adaa6 --- /dev/null +++ b/packages/api/src/lib/date-filters.ts @@ -0,0 +1,45 @@ +/** + * Date filter bounds from a request (issue #104). Collection dates are stored as YYYY-MM-DD and + * compared as strings, so a malformed bound such as `garbage` or `5/7/2024` would silently + * match everything or nothing. These helpers reject it with a `ValidationError` (400) instead. + */ +import { ValidationError } from './error-handler' +import { parseCollectionDate } from './validation' + +const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/ +const TIMESTAMP = /^(\d{4}-\d{2}-\d{2})(?:[T ](\d{2}:\d{2}(?::\d{2})?))?$/ + +/** True when `value` is a real calendar date written YYYY-MM-DD. */ +export function isIsoCalendarDate(value: string): boolean { + return ISO_DATE.test(value) && parseCollectionDate(value) === value +} + +/** + * A collection-date filter bound. Empty or omitted means no bound; anything other than a real + * YYYY-MM-DD date throws a `ValidationError` naming `label`. + */ +export function collectionDateFilter(label: string, value: string | null | undefined): string | undefined { + if (value == null) return undefined + const text = value.trim() + if (text === '') return undefined + if (!isIsoCalendarDate(text)) { + throw new ValidationError(`Invalid ${label} '${value}': use a date written YYYY-MM-DD.`) + } + return text +} + +/** + * A bound on a created timestamp (stored as `YYYY-MM-DD HH:MM:SS`): YYYY-MM-DD, optionally + * followed by a time. A `T` separator is stored as a space so it compares with stored values. + * Empty or omitted means no bound; anything else throws a `ValidationError` naming `label`. + */ +export function timestampFilter(label: string, value: string | null | undefined): string | undefined { + if (value == null) return undefined + const text = value.trim() + if (text === '') return undefined + const match = TIMESTAMP.exec(text) + if (!match || !isIsoCalendarDate(match[1])) { + throw new ValidationError(`Invalid ${label} '${value}': use YYYY-MM-DD or YYYY-MM-DD HH:MM:SS.`) + } + return match[2] ? `${match[1]} ${match[2]}` : match[1] +} diff --git a/packages/api/src/lib/derivations.ts b/packages/api/src/lib/derivations.ts index 7855e70f..026a2b8c 100644 --- a/packages/api/src/lib/derivations.ts +++ b/packages/api/src/lib/derivations.ts @@ -14,6 +14,7 @@ import { and, eq, isNull, sql } from 'drizzle-orm' import { validateContainerTypeForSpecimenType } from '../lib/validation' import { getDefaultUnit } from './defaults' import { utcNow } from './datetime' +import { isIsoCalendarDate } from './date-filters' import { createContainerForSpecimen, pickContainerQuantity } from './container-creation' import { withWriteTransaction } from '../db/write-transaction' import { insertedRow } from './inserted-row' @@ -67,6 +68,13 @@ async function findOrCreateDerivedSpecimen( if (!parentSpecimen) { throw new Error('Parent specimen not found') } + // The child is the same collection event, so it takes the parent's date as stored. Refuse to + // copy a date that isn't YYYY-MM-DD rather than spread it to another Specimen (issue #104). + if (parentSpecimen.collectionDate != null && !isIsoCalendarDate(parentSpecimen.collectionDate)) { + throw new Error( + `Parent specimen ${parentSpecimen.id} has collection date '${parentSpecimen.collectionDate}', which is not YYYY-MM-DD. Correct it before deriving from it.`, + ) + } const type = await database .select() diff --git a/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts b/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts index ed8c8110..d28f703f 100644 --- a/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts +++ b/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts @@ -195,6 +195,38 @@ describe('resolveParentContainerId', () => { }) }) + describe('parent_collection_date', () => { + async function seedDatedPaperParent() { + const study = await createTestStudy(testDb, { title: 'Study', shortCode: 'STD2' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj-2' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Whole Blood' }) + const spec = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id, collectionDate: '2024-05-07' }) + const container = await createTestStorageContainer(testDb, { specimenId: spec.id }) + await testDb.insert(paper).values({ id: container.id, sheetId: await createSheet(testDb), sublabel: 'Spot-B' }) + return container.id + } + + function datedRow(parentCollectionDate: string) { + return row({ + parent_container_type: 'paper', + parent_study_short_code: 'STD2', + parent_subject_name: 'Subj-2', + parent_specimen_type_name: 'Whole Blood', + parent_collection_date: parentCollectionDate, + }) + } + + it.each(['2024-05-07', '5/7/2024'])('matches the stored date when written %s', async (date) => { + const containerId = await seedDatedPaperParent() + expect(await resolveParentContainerId(testDb, datedRow(date))).toBe(containerId) + }) + + it.each(['garbage', '5/7/24'])('rejects %s instead of matching nothing', async (date) => { + await seedDatedPaperParent() + await expect(resolveParentContainerId(testDb, datedRow(date))).rejects.toThrow(`Invalid parent_collection_date '${date}'`) + }) + }) + it('throws when no parent identifier is provided', async () => { await expect(resolveParentContainerId(testDb, row({}))).rejects.toThrow( 'Unable to resolve parent container', diff --git a/packages/api/src/lib/derivations/parent-container-resolver.ts b/packages/api/src/lib/derivations/parent-container-resolver.ts index faa446ef..489f0540 100644 --- a/packages/api/src/lib/derivations/parent-container-resolver.ts +++ b/packages/api/src/lib/derivations/parent-container-resolver.ts @@ -12,6 +12,7 @@ import { } from '../../db/schema' import type { DatabaseOrTransaction } from '../db-types' import { resolveContainerByBarcode } from '../identifier-resolution' +import { parseCollectionDate } from '../validation' import type { DerivationCsvRow } from '../derivations-csv' function parseNumber(value?: string): number | undefined { @@ -20,6 +21,18 @@ function parseNumber(value?: string): number | undefined { return Number.isNaN(n) ? undefined : n } +/** + * `parent_collection_date` as stored (YYYY-MM-DD), so a date typed M/D/YYYY still matches. + * Blank means no date filter; a value that isn't a date is an error rather than no match. + */ +function parentCollectionDate(row: DerivationCsvRow): string | undefined { + const raw = (row.parent_collection_date ?? '').trim() + if (raw === '') return undefined + const date = parseCollectionDate(raw) + if (!date) throw new Error(`Invalid parent_collection_date '${raw}'. Use YYYY-MM-DD (or M/D/YYYY).`) + return date +} + /** * Resolve the parent container for a derivation CSV row to a storage-container id. * @@ -57,6 +70,8 @@ export async function resolveParentContainerId( throw new Error(`Parent container barcode '${barcode}' not found`) } + const parentDate = parentCollectionDate(row) + // 3. Control batch identification if (row.parent_control_batch_name || row.parent_control_batch_id) { if (!row.parent_specimen_type_name) { @@ -114,15 +129,12 @@ export async function resolveParentContainerId( const candidates = await database .select({ id: specimen.id }) .from(specimen) - .where(row.parent_collection_date - ? and(where, eq(specimen.collectionDate, row.parent_collection_date.trim())) as any - : where, - ) + .where(parentDate ? and(where, eq(specimen.collectionDate, parentDate)) as any : where) if (candidates.length === 0) { throw new Error(`No ${row.parent_specimen_type_name} specimen found in control batch '${row.parent_control_batch_name || row.parent_control_batch_id}'`) } - if (candidates.length > 1 && !row.parent_collection_date) { + if (candidates.length > 1 && !parentDate) { throw new Error(`Multiple ${row.parent_specimen_type_name} specimens found in batch; add parent_collection_date to disambiguate`) } @@ -263,10 +275,7 @@ export async function resolveParentContainerId( const candidates = await database .select({ id: specimen.id }) .from(specimen) - .where(row.parent_collection_date - ? and(where, eq(specimen.collectionDate, row.parent_collection_date.trim())) as any - : where, - ) + .where(parentDate ? and(where, eq(specimen.collectionDate, parentDate)) as any : where) if (candidates.length === 0) { throw new Error('No matching parent specimen/paper container found for subject/specimen criteria') diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 327b24a9..a6e8fe3b 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -3,6 +3,7 @@ import { specimen, storageContainer, studySubject, study, specimenType, controlB import { eq, and, or, sql } from 'drizzle-orm' import { validatePage, validateLimit } from '../constants' import { resolveContainerByBarcode } from '../identifier-resolution' +import { collectionDateFilter, timestampFilter } from '../date-filters' import { dateToUpperBound } from '../sql-helpers' import { likeContains } from '../sql-like' @@ -142,17 +143,21 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer } } - if (collectionDateFrom) { - conditions.push(sql`${specimen.collectionDate} >= ${collectionDateFrom}`) + const collectionFrom = collectionDateFilter('collection_date_from', collectionDateFrom) + const collectionTo = collectionDateFilter('collection_date_to', collectionDateTo) + const createdFromBound = timestampFilter('created_from', createdFrom) + const createdToBound = timestampFilter('created_to', createdTo) + if (collectionFrom) { + conditions.push(sql`${specimen.collectionDate} >= ${collectionFrom}`) } - if (collectionDateTo) { - conditions.push(sql`${specimen.collectionDate} <= ${collectionDateTo}`) + if (collectionTo) { + conditions.push(sql`${specimen.collectionDate} <= ${collectionTo}`) } - if (createdFrom) { - conditions.push(sql`${specimen.created} >= ${createdFrom}`) + if (createdFromBound) { + conditions.push(sql`${specimen.created} >= ${createdFromBound}`) } - if (createdTo) { - conditions.push(dateToUpperBound(specimen.created, createdTo)) + if (createdToBound) { + conditions.push(dateToUpperBound(specimen.created, createdToBound)) } if (barcode) { diff --git a/packages/api/src/lib/statistics/parse-filters.ts b/packages/api/src/lib/statistics/parse-filters.ts index f8dc3b06..9210fe79 100644 --- a/packages/api/src/lib/statistics/parse-filters.ts +++ b/packages/api/src/lib/statistics/parse-filters.ts @@ -1,4 +1,5 @@ import type { Context } from 'hono' +import { collectionDateFilter, timestampFilter } from '../date-filters' import type { StatisticsFilters } from './types' /** Parse an integer id query value; anything else means no filter. */ @@ -8,7 +9,10 @@ function parseId(value: string | undefined): number | undefined { return isNaN(id) ? undefined : id } -/** Parse dashboard statistics query parameters from a Hono request. */ +/** + * Parse dashboard statistics query parameters from a Hono request. A malformed date bound + * throws a `ValidationError` (400). + */ export function parseStatisticsFilters(c: Context): StatisticsFilters { const tagIdsParam = c.req.queries('tag_ids') || c.req.queries('tag_ids[]') const tagIds = tagIdsParam?.map((id) => parseInt(id)).filter((id) => !isNaN(id)) @@ -20,9 +24,9 @@ export function parseStatisticsFilters(c: Context): StatisticsFilters { container_type: c.req.query('container_type'), tag_ids: tagIds, location_id: parseId(c.req.query('location_id')), - collection_date_from: c.req.query('collection_date_from'), - collection_date_to: c.req.query('collection_date_to'), - created_from: c.req.query('created_from'), - created_to: c.req.query('created_to'), + collection_date_from: collectionDateFilter('collection_date_from', c.req.query('collection_date_from')), + collection_date_to: collectionDateFilter('collection_date_to', c.req.query('collection_date_to')), + created_from: timestampFilter('created_from', c.req.query('created_from')), + created_to: timestampFilter('created_to', c.req.query('created_to')), } } diff --git a/packages/api/src/routes/__tests__/data-audit.test.ts b/packages/api/src/routes/__tests__/data-audit.test.ts index 628626f3..104413fe 100644 --- a/packages/api/src/routes/__tests__/data-audit.test.ts +++ b/packages/api/src/routes/__tests__/data-audit.test.ts @@ -245,6 +245,7 @@ describe('Data audit API', () => { duplicateBarcodes: unknown[] locationPathInconsistencies: unknown[] containersWithNoGridPosition: unknown[] + specimensWithNonIsoCollectionDate: unknown[] } expect(Array.isArray(data.emptyCollections)).toBe(true) expect(Array.isArray(data.collectionsWithMissingLocation)).toBe(true) @@ -258,6 +259,25 @@ describe('Data audit API', () => { expect(Array.isArray(data.duplicateBarcodes)).toBe(true) expect(Array.isArray(data.locationPathInconsistencies)).toBe(true) expect(Array.isArray(data.containersWithNoGridPosition)).toBe(true) + expect(Array.isArray(data.specimensWithNonIsoCollectionDate)).toBe(true) + }) + + it('flags specimens whose collection date is not YYYY-MM-DD', async () => { + const specType = await createTestSpecimenType(ctx.db, { name: 'STDate' }) + const iso = await createTestSpecimen(ctx.db, specType.id, { collectionDate: '2024-05-07' }) + const legacy = await createTestSpecimen(ctx.db, specType.id, { collectionDate: '5/7/24' }) + const impossible = await createTestSpecimen(ctx.db, specType.id, { collectionDate: '2024-02-30' }) + await createTestSpecimen(ctx.db, specType.id) + + const res = await ctx.request('/api/admin/data-audit/integrity-report', { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { specimensWithNonIsoCollectionDate: Array<{ id: number; collectionDate: string }> } + expect(data.specimensWithNonIsoCollectionDate).toEqual([ + { id: legacy.id, collectionDate: '5/7/24' }, + { id: impossible.id, collectionDate: '2024-02-30' }, + ]) + expect(data.specimensWithNonIsoCollectionDate.map((s) => s.id)).not.toContain(iso.id) }) it('includes containers with missing specimen in report', async () => { diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 3e374370..2508cf10 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -125,6 +125,22 @@ describe('Export API', () => { expect(rowsAfter.length).toBe(rowsBefore.length) }) + it.each([ + ['date_from', 'garbage'], + ['date_to', '5/7/2024'], + ['date_from', '2024'], + ['created_from', 'yesterday'], + ])('returns 400 for %s=%s', async (param, value) => { + await seedTypedContainers('GDATE', { cryovial: false }) + + const res = await ctx.request(`/api/export/containers?study=GDATE&count_only=true&${param}=${encodeURIComponent(value)}`, { + method: 'GET', + }) + + expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ errorCode: 'VALIDATION_ERROR' }) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/export/containers?study=ST1', { method: 'GET' }) expect(res.status).toBe(401) @@ -442,6 +458,8 @@ describe('Export API', () => { ['an empty entry', { Subj1: {} }], ['a non-object map', 'Subj1'], ['a non-string date', { Subj1: { exact: 20240115 } }], + ['an exact date that is not YYYY-MM-DD', { Subj1: { exact: '1/15/2024' } }], + ['a range bound that is not a date', { Subj1: [{ from: 'garbage' }] }], ])('rejects %s with 400', async (_label, subjectDates) => { await seedSubjectWithSpecimen() @@ -512,6 +530,27 @@ describe('Export API', () => { expect(typeof data.data).toBe('string') expect(data.summary).toBeDefined() }) + + it.each([ + ['an entry collection_date', { collection_date: '1/15/2024' }, {}], + ['an entry date_to', { date_to: 'garbage' }, {}], + ['the global date_from', {}, { date_from: '2024' }], + ])('returns 400 for a malformed date in %s', async (_label, entryDates, globalDates) => { + const study = await createTestStudy(ctx.db, { title: 'Multi', shortCode: 'MULTI2' }) + await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Alice' }) + + const res = await ctx.request('/api/export/containers/multi-study', { + method: 'POST', + json: { + entries: [{ study_short_code: 'MULTI2', subject_name: 'Alice', ...entryDates }], + ...globalDates, + count_only: true, + }, + }) + + expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ errorCode: 'VALIDATION_ERROR' }) + }) }) describe('POST /api/export/containers/by-barcodes', () => { diff --git a/packages/api/src/routes/__tests__/specimens.test.ts b/packages/api/src/routes/__tests__/specimens.test.ts index b13fc9fb..1eb864d1 100644 --- a/packages/api/src/routes/__tests__/specimens.test.ts +++ b/packages/api/src/routes/__tests__/specimens.test.ts @@ -66,6 +66,33 @@ describe('Specimens API', () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/specimens', { method: 'GET' }) expect(res.status).toBe(401) }) + + it('filters by collection date as a calendar date', async () => { + const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) + const early = await createTestSpecimen(ctx.db, type.id, { collectionDate: '1999-12-31' }) + await createTestSpecimen(ctx.db, type.id, { collectionDate: '2024-05-07' }) + + const res = await ctx.request('/api/specimens?collection_date_to=2000-01-01') + + expect(res.status).toBe(200) + const data = (await res.json()) as { specimens: Array<{ id: number }> } + expect(data.specimens.map((s) => s.id)).toEqual([early.id]) + }) + + it.each([ + ['collection_date_from', 'garbage'], + ['collection_date_to', '5/7/2024'], + ['collection_date_from', '2024'], + ['collection_date_to', '2024-02-30'], + ['created_from', 'yesterday'], + ['created_to', '2024-13-01'], + ])('returns 400 for %s=%s', async (param, value) => { + const res = await ctx.request(`/api/specimens?${param}=${encodeURIComponent(value)}`) + + expect(res.status).toBe(400) + const data = (await res.json()) as { error: string } + expect(data.error).toContain(param) + }) }) describe('POST /api/specimens/bulk', () => { @@ -229,6 +256,44 @@ describe('Specimens API', () => { expect(after.length).toBe(before.length) }) + it('POST /api/specimens stores an M/D/YYYY collection date as YYYY-MM-DD', async () => { + const res = await ctx.request('/api/specimens', { + method: 'POST', + json: { + sourceType: 'subject', + studyShortCode: 'BLK01', + subjectName: 'BULK-SUBJ', + specimenTypeName: 'Whole Blood', + collectionDate: '5/7/2024', + container: { containerType: 'cryovial_tube', collection: { type: 'cryovial_box', name: 'BULK-BOX', position: 'A01' } }, + }, + }) + + expect(res.status).toBe(201) + const rows = await ctx.db.select().from(specimen).where(eq(specimen.studySubjectId, testSubject.id)) + expect(rows.map((r) => r.collectionDate)).toEqual(['2024-05-07']) + }) + + it.each(['5/7/24', 'garbage', '2024-02-30'])( + 'POST /api/specimens rejects collection date %s and stores nothing', + async (collectionDate) => { + const res = await ctx.request('/api/specimens', { + method: 'POST', + json: { + sourceType: 'subject', + studyShortCode: 'BLK01', + subjectName: 'BULK-SUBJ', + specimenTypeName: 'Whole Blood', + collectionDate, + container: { containerType: 'cryovial_tube', collection: { type: 'cryovial_box', name: 'BULK-BOX', position: 'A02' } }, + }, + }) + + expect(res.status).toBe(400) + expect(await ctx.db.select().from(specimen).where(eq(specimen.studySubjectId, testSubject.id))).toEqual([]) + }, + ) + it('rolls back single specimen creation when container creation fails', async () => { const beforeSpecimens = await ctx.db.select({ id: specimen.id }).from(specimen) diff --git a/packages/api/src/routes/__tests__/statistics.test.ts b/packages/api/src/routes/__tests__/statistics.test.ts index 7b69e89b..a41ddc60 100644 --- a/packages/api/src/routes/__tests__/statistics.test.ts +++ b/packages/api/src/routes/__tests__/statistics.test.ts @@ -5,7 +5,7 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createStatisticsRoutes } from '../statistics' -import { createTestSpecimenType, createTestStorageContainer } from '../../__tests__/helpers/factories' +import { createTestSpecimen, createTestSpecimenType, createTestStorageContainer } from '../../__tests__/helpers/factories' describe('Statistics API', () => { let ctx: AuthenticatedRouteTestContext @@ -62,6 +62,30 @@ describe('Statistics API', () => { expect(await total('?specimen_type_id=abc&location_id=abc')).toBe(1) }) + it.each([ + ['collection_date_from', 'garbage'], + ['collection_date_to', '9/7/2022'], + ['collection_date_from', '2024'], + ['created_to', 'soon'], + ])('returns 400 for %s=%s, even for an unknown study', async (param, value) => { + for (const study of ['', '&study=NONEXISTENT']) { + const res = await ctx.request(`/api/statistics?${param}=${encodeURIComponent(value)}${study}`, { method: 'GET' }) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + } + }) + + it('filters by collection date as a calendar date', async () => { + const type = await createTestSpecimenType(ctx.db, { name: 'Dated' }) + await createTestSpecimen(ctx.db, type.id, { collectionDate: '1999-12-31' }) + await createTestSpecimen(ctx.db, type.id, { collectionDate: '2024-05-07' }) + + const res = await ctx.request('/api/statistics?collection_date_to=2000-01-01', { method: 'GET' }) + + expect(res.status).toBe(200) + expect(((await res.json()) as { specimens: { total: number } }).specimens.total).toBe(1) + }) + it('returns 401 without auth', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/statistics', { method: 'GET' }) expect(res.status).toBe(401) diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index a2ea1f23..91b131be 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -29,6 +29,7 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Storage container tag orphans.** Tag links that reference a deleted container or tag. - **Duplicate barcodes (micronix).** Micronix tubes that share the same barcode (schema expects unique barcodes). Duplicate barcodes on cryovial tubes are **not** treated as a data integrity issue. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. +- **Collection date not ISO.** Specimens whose collection date is not a real date written `YYYY-MM-DD`. Date filters, exports, and study summaries compare collection dates as text, so these Specimens are missed or placed in the wrong range. Correct the date in the database. - **Containers with no grid position.** Tubes or wells in a collection with no well or slot. Legacy rows are allowed and do **not** count toward the overview issue total. Assign a position via edit or scan move when the well is known. ## Empty collections diff --git a/packages/web/src/lib/api/admin.ts b/packages/web/src/lib/api/admin.ts index 19b57db1..ac101e55 100644 --- a/packages/web/src/lib/api/admin.ts +++ b/packages/web/src/lib/api/admin.ts @@ -87,6 +87,11 @@ export interface ContainerWithNoGridPositionItem { collectionId: number } +export interface SpecimenWithNonIsoCollectionDateItem { + id: number + collectionDate: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -100,6 +105,7 @@ export interface IntegrityReport { duplicateBarcodes: DuplicateBarcodeItem[] locationPathInconsistencies: LocationPathInconsistencyItem[] containersWithNoGridPosition: ContainerWithNoGridPositionItem[] + specimensWithNonIsoCollectionDate: SpecimenWithNonIsoCollectionDateItem[] } export interface CollectionWithMissingLocationItem { diff --git a/packages/web/src/pages/AdminDataIntegrityOverview.tsx b/packages/web/src/pages/AdminDataIntegrityOverview.tsx index 5d1b2d9b..2384f1ad 100644 --- a/packages/web/src/pages/AdminDataIntegrityOverview.tsx +++ b/packages/web/src/pages/AdminDataIntegrityOverview.tsx @@ -39,7 +39,8 @@ export default function AdminDataIntegrityOverview() { report.derivationBrokenRefs.length + report.storageContainerTagOrphans.length + report.duplicateBarcodes.length + - report.locationPathInconsistencies.length + report.locationPathInconsistencies.length + + report.specimensWithNonIsoCollectionDate.length // containersWithNoGridPosition is informational and is not an integrity issue count return ( diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index d7e44eb2..cd8f3853 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -386,6 +386,35 @@ export default function AdminDataIntegrityReport() {
+ +
+ + + + + + + + + {report.specimensWithNonIsoCollectionDate.map((s) => ( + + + + + ))} + +
Specimen IDCollection date
+ + {s.id} + + {s.collectionDate}
+
+ + { }) }) + it('counts specimens whose collection date is not ISO as integrity issues', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + specimensWithNonIsoCollectionDate: [ + { id: 1, collectionDate: '5/7/24' }, + { id: 2, collectionDate: '9/14/2020' }, + ], + }) + await render() + await waitFor(() => { + expect(screen.getByRole('link', { name: /integrity report/i })).toHaveTextContent('2') + }) + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() diff --git a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx index c2baf923..b6ae3fbe 100644 --- a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx +++ b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx @@ -17,6 +17,7 @@ function emptyIntegrityReport() { duplicateBarcodes: [], locationPathInconsistencies: [], containersWithNoGridPosition: [], + specimensWithNonIsoCollectionDate: [], } } @@ -56,6 +57,17 @@ describe('AdminDataIntegrityReport', () => { }) }) + it('shows specimens whose collection date is not ISO', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + specimensWithNonIsoCollectionDate: [{ id: 143200, collectionDate: '9/14/20' }], + }) + await render() + expect(await screen.findByText('Collection date not ISO')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '143200' })).toHaveAttribute('href', '/specimens/143200') + expect(screen.getByText('9/14/20')).toBeInTheDocument() + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() From 67481abb608c5c9222c37d5e5bbf6485d9b22265 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:38:46 -0700 Subject: [PATCH 09/54] fix(web): parse server timestamps as UTC (#117) The API stores UTC timestamps as SQLite `YYYY-MM-DD HH:MM:SS` with no zone marker, and the web parsed them as local time. West of UTC a plate created just after midnight UTC showed the next day and a negative age. parseDisplayDate is now the one parser for dates from the API: a zoneless timestamp (space or T separator) is UTC, a bare YYYY-MM-DD or YYYY-MM is a local calendar date, and anything with Z or an offset keeps its zone. Every created, last_updated, last_login, error log and activity timestamp now goes through it (or the new displayDateTime / formatDateTime helpers). Relative ages clamp at zero so clock skew reads "Just now". Statistics and dashboard timeline months no longer shift back a month in the Americas. Co-Authored-By: Claude Opus 5.5 --- packages/web/src/components/StudyCard.tsx | 8 +- .../web/src/components/SubjectMergeModal.tsx | 5 +- .../__tests__/ActivityFeed.test.tsx | 33 ++++++- .../components/__tests__/ContentCard.test.tsx | 31 +++++++ .../src/components/dashboard/ActivityFeed.tsx | 8 +- .../components/dashboard/RecentStudies.tsx | 3 +- .../components/dashboard/SystemInsights.tsx | 9 +- .../web/src/config/reference-data-config.ts | 3 +- packages/web/src/hooks/useDashboard.ts | 3 +- .../web/src/lib/__tests__/date-utils.test.ts | 88 +++++++++++++++++++ packages/web/src/lib/date-utils.ts | 44 ++++++++-- packages/web/src/pages/AdminErrorLogs.tsx | 3 +- packages/web/src/pages/AdminUsers.tsx | 5 +- packages/web/src/pages/ContainerDetail.tsx | 6 +- packages/web/src/pages/LocationDetail.tsx | 5 +- packages/web/src/pages/Locations.tsx | 5 +- packages/web/src/pages/SpecimenDetail.tsx | 4 +- packages/web/src/pages/Statistics.tsx | 14 +-- packages/web/src/pages/Studies.tsx | 3 +- packages/web/src/pages/StudyDetail.tsx | 5 +- 20 files changed, 240 insertions(+), 45 deletions(-) create mode 100644 packages/web/src/components/__tests__/ContentCard.test.tsx diff --git a/packages/web/src/components/StudyCard.tsx b/packages/web/src/components/StudyCard.tsx index 037ab515..84678291 100644 --- a/packages/web/src/components/StudyCard.tsx +++ b/packages/web/src/components/StudyCard.tsx @@ -1,6 +1,6 @@ import { useState } from 'react' import { Link } from 'react-router-dom' -import { formatDate } from '../lib/date-utils' +import { formatDate, parseDisplayDate } from '../lib/date-utils' import type { Study, StudySummaryBasic } from '../lib/api/studies';interface StudySummaryData extends StudySummaryBasic { averageSpecimensPerSubject?: number studyDurationDays?: number | null @@ -19,8 +19,8 @@ interface StudyCardProps { // Calculate study duration from date range const calculateDuration = (range: { earliest: string; latest: string } | null): number | null => { if (!range) return null - const start = new Date(range.earliest) - const end = new Date(range.latest) + const start = parseDisplayDate(range.earliest) + const end = parseDisplayDate(range.latest) const diffTime = Math.abs(end.getTime() - start.getTime()) return Math.ceil(diffTime / (1000 * 60 * 60 * 24)) } @@ -263,7 +263,7 @@ export default function StudyCard({ study, summary, loading, onLoadSummary, vari {/* Footer */}
- Updated {new Date(study.lastUpdated).toLocaleDateString()} + Updated {parseDisplayDate(study.lastUpdated).toLocaleDateString()} e.stopPropagation()} diff --git a/packages/web/src/components/SubjectMergeModal.tsx b/packages/web/src/components/SubjectMergeModal.tsx index 51bcd788..1455133d 100644 --- a/packages/web/src/components/SubjectMergeModal.tsx +++ b/packages/web/src/components/SubjectMergeModal.tsx @@ -4,6 +4,7 @@ import { studiesApi } from '../lib/api/studies'; import type { StudySubject } from '../lib/api/types'; import { useClickOutside } from '../hooks/useClickOutside' import { Modal } from '../ui' +import { parseDisplayDate } from '../lib/date-utils' interface SubjectMergeModalProps { isOpen: boolean @@ -315,7 +316,7 @@ function SubjectMergeModalContent({ Specimens: {sourceSpecimenCount ?? '...'}
- Created: {new Date(sourceSubject.created).toLocaleDateString()} + Created: {parseDisplayDate(sourceSubject.created).toLocaleDateString()}
)} @@ -442,7 +443,7 @@ function SubjectMergeModalContent({ Specimens: {targetSpecimenCount ?? '...'}
- Created: {new Date(targetSubject.created).toLocaleDateString()} + Created: {parseDisplayDate(targetSubject.created).toLocaleDateString()}
)} diff --git a/packages/web/src/components/__tests__/ActivityFeed.test.tsx b/packages/web/src/components/__tests__/ActivityFeed.test.tsx index 5077b1f5..c3c507c0 100644 --- a/packages/web/src/components/__tests__/ActivityFeed.test.tsx +++ b/packages/web/src/components/__tests__/ActivityFeed.test.tsx @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest' +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import { render, screen } from '../../__tests__/helpers/render' import ActivityFeed from '../dashboard/ActivityFeed' @@ -47,3 +47,34 @@ describe('ActivityFeed', () => { expect(html).not.toMatch(/#\d+/) }) }) + +describe('ActivityFeed server timestamps', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }) + // 6:20 PM PDT on Oct 31, 2026 (vitest.config.ts runs in America/Los_Angeles) + vi.setSystemTime(new Date('2026-11-01T01:20:02Z')) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('reads a timestamp just after midnight UTC as UTC: today, minutes ago', async () => { + await render( + , + ) + expect(screen.getByText('Today')).toBeInTheDocument() + expect(screen.getByText('10 minutes ago')).toBeInTheDocument() + }) + + it('never shows a negative age for a timestamp ahead of the browser clock', async () => { + await render( + , + ) + expect(screen.getByText('Just now')).toBeInTheDocument() + }) +}) diff --git a/packages/web/src/components/__tests__/ContentCard.test.tsx b/packages/web/src/components/__tests__/ContentCard.test.tsx new file mode 100644 index 00000000..0a6fac43 --- /dev/null +++ b/packages/web/src/components/__tests__/ContentCard.test.tsx @@ -0,0 +1,31 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { render, screen } from '../../__tests__/helpers/render' +import ContentCard from '../ContentCard' + +// vitest.config.ts runs the suite in America/Los_Angeles (UTC-7 on this date). +describe('ContentCard timestamps', () => { + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'] }) + // 6:20 PM PDT on Oct 31, 2026 + vi.setSystemTime(new Date('2026-11-01T01:20:02Z')) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('shows a plate created just after midnight UTC on its local day, with a past age', async () => { + await render( + , + ) + expect(screen.getByText('Created: Oct 31, 2026')).toBeInTheDocument() + expect(screen.getByText('Updated: 10 minutes ago')).toBeInTheDocument() + }) +}) diff --git a/packages/web/src/components/dashboard/ActivityFeed.tsx b/packages/web/src/components/dashboard/ActivityFeed.tsx index fd8f6c1f..ffac22a7 100644 --- a/packages/web/src/components/dashboard/ActivityFeed.tsx +++ b/packages/web/src/components/dashboard/ActivityFeed.tsx @@ -1,5 +1,6 @@ import { Link } from 'react-router-dom' import { SkeletonList } from '../../ui' +import { parseDisplayDate } from '../../lib/date-utils' interface ActivityItem { id: number @@ -15,9 +16,10 @@ interface ActivityFeedProps { } function formatRelativeTime(timestamp: string): string { - const date = new Date(timestamp) + const date = parseDisplayDate(timestamp) const now = new Date() - const diffMs = now.getTime() - date.getTime() + // Clock skew can put a server timestamp slightly ahead of this clock; never show a negative age. + const diffMs = Math.max(0, now.getTime() - date.getTime()) const diffMins = Math.floor(diffMs / 60000) const diffHours = Math.floor(diffMs / 3600000) const diffDays = Math.floor(diffMs / 86400000) @@ -147,7 +149,7 @@ function groupActivitiesByDate(activities: ActivityItem[]): { } activities.forEach((activity) => { - const activityDate = new Date(activity.timestamp) + const activityDate = parseDisplayDate(activity.timestamp) if (activityDate >= today) { grouped.today.push(activity) } else if (activityDate >= yesterday) { diff --git a/packages/web/src/components/dashboard/RecentStudies.tsx b/packages/web/src/components/dashboard/RecentStudies.tsx index 2a5042f4..77c5f6df 100644 --- a/packages/web/src/components/dashboard/RecentStudies.tsx +++ b/packages/web/src/components/dashboard/RecentStudies.tsx @@ -1,6 +1,7 @@ import { Link } from 'react-router-dom' import { Study, StudySummaryBasic } from '../../lib/api/studies'; import { SkeletonList } from '../../ui' +import { parseDisplayDate } from '../../lib/date-utils' interface RecentStudiesProps { studies: Array @@ -61,7 +62,7 @@ export default function RecentStudies({ studies, loading }: RecentStudiesProps)
- {new Date(study.lastUpdated).toLocaleDateString()} + {parseDisplayDate(study.lastUpdated).toLocaleDateString()}
diff --git a/packages/web/src/components/dashboard/SystemInsights.tsx b/packages/web/src/components/dashboard/SystemInsights.tsx index e4025b16..f70ac371 100644 --- a/packages/web/src/components/dashboard/SystemInsights.tsx +++ b/packages/web/src/components/dashboard/SystemInsights.tsx @@ -5,6 +5,7 @@ import { StatisticsData } from '../../lib/api/statistics'; import { SkeletonCard } from '../../ui' import { useTheme } from '../../contexts/ThemeContext' import { getAppChartColors } from '../../lib/chart-colors' +import { parseDisplayDate, toLocalDateString } from '../../lib/date-utils' interface SystemInsightsProps { data: StatisticsData | null @@ -64,11 +65,11 @@ export default function SystemInsights({ data, loading }: SystemInsightsProps) { // Get raw timeline data const rawTimelineData = data.specimens.collectionTimeline - .filter((item) => new Date(item.date) >= ninetyDaysAgo) + .filter((item) => parseDisplayDate(item.date) >= ninetyDaysAgo) .map((item) => ({ - name: new Date(item.date).toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), + name: parseDisplayDate(item.date).toLocaleDateString('en-US', { month: 'short', day: 'numeric' }), value: item.count, - date: new Date(item.date).getTime(), + date: parseDisplayDate(item.date).getTime(), })) .sort((a, b) => a.date - b.date) @@ -80,7 +81,7 @@ export default function SystemInsights({ data, loading }: SystemInsightsProps) { rawTimelineData.forEach((item) => { const weekStart = new Date(item.date) weekStart.setDate(weekStart.getDate() - weekStart.getDay()) // Start of week - const weekKey = weekStart.toISOString().split('T')[0] + const weekKey = toLocalDateString(weekStart) const existing = weekMap.get(weekKey) if (existing) { diff --git a/packages/web/src/config/reference-data-config.ts b/packages/web/src/config/reference-data-config.ts index 073e3343..9950d5a1 100644 --- a/packages/web/src/config/reference-data-config.ts +++ b/packages/web/src/config/reference-data-config.ts @@ -6,6 +6,7 @@ import type { Tag, StorageType, Strain } from '../lib/api/reference-data'; import { specimenTypesApi, tagsApi, storageTypesApi, strainsApi, unitsApi } from '../lib/api/reference-data'; import ContainerTypesCell from '../components/ContainerTypesCell' import ContainerTypeToggle from '../components/ContainerTypeToggle' +import { parseDisplayDate } from '../lib/date-utils' export type ReferenceDataType = | 'specimen-types' @@ -94,7 +95,7 @@ export const referenceDataConfigs: ReferenceDataConfig[] = [ { key: 'created', label: 'Created', - render: (value: string) => new Date(value).toLocaleDateString(), + render: (value: string) => parseDisplayDate(value).toLocaleDateString(), }, ] }, diff --git a/packages/web/src/hooks/useDashboard.ts b/packages/web/src/hooks/useDashboard.ts index 243dfa02..928a20e0 100644 --- a/packages/web/src/hooks/useDashboard.ts +++ b/packages/web/src/hooks/useDashboard.ts @@ -9,6 +9,7 @@ import { activityApi } from '../lib/api/search' import { statisticsApi, type StatisticsData } from '../lib/api/statistics' import { controlsApi } from '../lib/api/controls' import { qpcrExperimentsApi, type QpcrExperiment } from '../lib/api/qpcr' +import { displayDateTime } from '../lib/date-utils' export type DashboardStats = { studies: number @@ -151,7 +152,7 @@ export function useDashboardQpcr() { const res = await qpcrExperimentsApi.list({ limit: 5 }) const all = res.experiments return [...all] - .sort((a, b) => new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime()) + .sort((a, b) => displayDateTime(b.lastUpdated) - displayDateTime(a.lastUpdated)) .slice(0, 5) }, }) diff --git a/packages/web/src/lib/__tests__/date-utils.test.ts b/packages/web/src/lib/__tests__/date-utils.test.ts index 19f7742d..e5b91979 100644 --- a/packages/web/src/lib/__tests__/date-utils.test.ts +++ b/packages/web/src/lib/__tests__/date-utils.test.ts @@ -91,3 +91,91 @@ describe('calendar dates in local time', () => { expect(toLocalDateString(new Date(2024, 0, 15, 23, 30))).toBe('2024-01-15') }) }) + +// vitest.config.ts runs the suite in America/Los_Angeles. On Oct 31, 2026 that is UTC-7 (PDT), +// so a server timestamp just after midnight UTC is still the previous evening locally. +describe('server timestamps (UTC with no zone marker)', () => { + // 6:20 PM PDT on Oct 31, 2026 + const now = new Date('2026-11-01T01:20:02Z') + + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(now) + }) + + afterEach(() => { + vi.useRealTimers() + }) + + it('runs in a time zone west of UTC', () => { + expect(now.getTimezoneOffset()).toBe(420) + }) + + it('parseDisplayDate reads a SQLite timestamp as UTC', async () => { + const { parseDisplayDate } = await import('../date-utils') + const d = parseDisplayDate('2026-11-01 01:10:02') + expect(d.toISOString()).toBe('2026-11-01T01:10:02.000Z') + expect([d.getFullYear(), d.getMonth(), d.getDate(), d.getHours()]).toEqual([2026, 9, 31, 18]) + }) + + it('parseDisplayDate gives the same instant for the T, Z and offset forms', async () => { + const { parseDisplayDate } = await import('../date-utils') + const expected = '2026-11-01T01:10:02.000Z' + expect(parseDisplayDate('2026-11-01T01:10:02').toISOString()).toBe(expected) + expect(parseDisplayDate('2026-11-01T01:10:02Z').toISOString()).toBe(expected) + expect(parseDisplayDate('2026-11-01T01:10:02.000Z').toISOString()).toBe(expected) + expect(parseDisplayDate('2026-10-31T18:10:02-07:00').toISOString()).toBe(expected) + expect(parseDisplayDate('2026-11-01 01:10').toISOString()).toBe('2026-11-01T01:10:00.000Z') + }) + + it('formatDate shows the local day for a timestamp just after midnight UTC', async () => { + const { formatDate } = await import('../date-utils') + expect(formatDate('2026-11-01 01:10:02')).toBe('Oct 31, 2026') + expect(formatDate('2026-11-01 00:00:00')).toBe('Oct 31, 2026') + }) + + it('formatDate shows the local day for a timestamp just before midnight UTC', async () => { + const { formatDate } = await import('../date-utils') + expect(formatDate('2026-10-31 23:59:59')).toBe('Oct 31, 2026') + // 7:30 AM UTC is still the previous local evening; 7:00 AM UTC is local midnight. + expect(formatDate('2026-10-31 06:59:59')).toBe('Oct 30, 2026') + expect(formatDate('2026-10-31 07:00:00')).toBe('Oct 31, 2026') + }) + + it('formatDateWithRelativeTime gives a past age for a timestamp just after midnight UTC', async () => { + const { formatDateWithRelativeTime } = await import('../date-utils') + expect(formatDateWithRelativeTime('2026-11-01 01:10:02')).toBe('10 minutes ago') + expect(formatDateWithRelativeTime('2026-10-31 23:20:02')).toBe('2 hours ago') + expect(formatDateWithRelativeTime('2026-10-31 01:20:02')).toBe('Yesterday') + }) + + it('formatDateWithRelativeTime never gives a negative age', async () => { + const { formatDateWithRelativeTime } = await import('../date-utils') + // A few seconds ahead of the browser clock (clock skew), and a day ahead. + expect(formatDateWithRelativeTime('2026-11-01 01:20:30')).toBe('Just now') + expect(formatDateWithRelativeTime('2026-11-02 01:20:02')).toBe('Just now') + }) + + it('formatDateTime shows a server timestamp in local time', async () => { + const { formatDateTime } = await import('../date-utils') + expect(formatDateTime('2026-11-01 01:10:02')).toBe( + new Date(2026, 9, 31, 18, 10, 2).toLocaleString(), + ) + expect(formatDateTime('2026-11-01 01:10:02', { dateStyle: 'medium', timeZone: 'America/Los_Angeles' })) + .toBe('Oct 31, 2026') + expect(formatDateTime(null)).toBe('N/A') + }) + + it('displayDateTime orders server timestamps across midnight UTC', async () => { + const { displayDateTime } = await import('../date-utils') + expect(displayDateTime('2026-11-01 00:00:01')).toBeGreaterThan(displayDateTime('2026-10-31 23:59:59')) + expect(displayDateTime('2026-11-01 00:00:01')).toBe(Date.UTC(2026, 10, 1, 0, 0, 1)) + }) + + it('does not shift a collection date or a timeline month', async () => { + const { formatDate, parseDisplayDate } = await import('../date-utils') + expect(formatDate('2026-11-01')).toBe('Nov 1, 2026') + const month = parseDisplayDate('2026-11') + expect([month.getFullYear(), month.getMonth(), month.getDate()]).toEqual([2026, 10, 1]) + }) +}) diff --git a/packages/web/src/lib/date-utils.ts b/packages/web/src/lib/date-utils.ts index 4002b00c..328e7f73 100644 --- a/packages/web/src/lib/date-utils.ts +++ b/packages/web/src/lib/date-utils.ts @@ -1,13 +1,32 @@ +const CALENDAR_DATE = /^(\d{4})-(\d{2})(?:-(\d{2}))?$/ +const ZONELESS_DATETIME = /^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?)$/ + /** - * Parse a date for display. A bare YYYY-MM-DD is a calendar date, so it is built in local - * time: new Date('2024-01-15') is UTC midnight, which renders as Jan 14 in the Americas. + * Parse a date or server timestamp for display. Every date string from the API goes through + * here. + * + * - A bare YYYY-MM-DD (a collection date) or YYYY-MM (a timeline month) is a calendar date, so + * it is built in local time: new Date('2024-01-15') is UTC midnight, which renders as Jan 14 + * in the Americas. + * - A timestamp with no zone marker, such as SQLite's `YYYY-MM-DD HH:MM:SS` from + * `current_timestamp` or `utcNow()`, is UTC. new Date() would read it as local time. + * - Anything else (ISO with `Z` or an offset) keeps its own zone. */ export function parseDisplayDate(value: string): Date { - const match = /^(\d{4})-(\d{2})-(\d{2})$/.exec(value) - if (match) return new Date(Number(match[1]), Number(match[2]) - 1, Number(match[3])) + const calendar = CALENDAR_DATE.exec(value) + if (calendar) { + return new Date(Number(calendar[1]), Number(calendar[2]) - 1, calendar[3] ? Number(calendar[3]) : 1) + } + const zoneless = ZONELESS_DATETIME.exec(value) + if (zoneless) return new Date(`${zoneless[1]}T${zoneless[2]}Z`) return new Date(value) } +/** Milliseconds since the epoch for a date or server timestamp (see parseDisplayDate). */ +export function displayDateTime(value: string): number { + return parseDisplayDate(value).getTime() +} + /** A Date as a local YYYY-MM-DD calendar date (toISOString() gives the UTC date). */ export function toLocalDateString(date: Date): string { const year = date.getFullYear() @@ -23,7 +42,7 @@ export function localToday(): string { /** * Formats a date string with relative time for recent dates and absolute dates for older items - * @param dateString - ISO date string + * @param dateString - Date or server timestamp (see parseDisplayDate) * @returns Formatted date string (e.g., "2 days ago" or "Jan 15, 2024") */ export function formatDateWithRelativeTime(dateString: string | null | undefined): string { @@ -31,7 +50,9 @@ export function formatDateWithRelativeTime(dateString: string | null | undefined const date = parseDisplayDate(dateString) const now = new Date() - const diffMs = now.getTime() - date.getTime() + // A timestamp slightly ahead of this clock (server clock skew) reads as "Just now", not as a + // negative age. + const diffMs = Math.max(0, now.getTime() - date.getTime()) const diffSeconds = Math.floor(diffMs / 1000) const diffMinutes = Math.floor(diffSeconds / 60) const diffHours = Math.floor(diffMinutes / 60) @@ -73,6 +94,17 @@ export function formatDate( return parseDisplayDate(dateString).toLocaleDateString('en-US', DATE_FORMATS[style]) } +/** + * Format a server timestamp as a local date and time, e.g. "1/15/2024, 3:45:00 PM" in en-US. + */ +export function formatDateTime( + dateString: string | null | undefined, + options?: Intl.DateTimeFormatOptions, +): string { + if (!dateString) return 'N/A' + return parseDisplayDate(dateString).toLocaleString(undefined, options) +} + /** * Format a date as a filesystem-safe local datetime string. * Format: YYYY-MM-DD_HH-MM-SS (e.g., "2026-01-27_14-30-45") diff --git a/packages/web/src/pages/AdminErrorLogs.tsx b/packages/web/src/pages/AdminErrorLogs.tsx index 29e0ccf6..3eae2b22 100644 --- a/packages/web/src/pages/AdminErrorLogs.tsx +++ b/packages/web/src/pages/AdminErrorLogs.tsx @@ -8,6 +8,7 @@ import Pagination from '../components/Pagination' import { useFocusSearchOnSlash } from '../hooks/useHotkey' import { Modal, PageError, fromQuery, getQueryErrorMessage } from '../ui' import '../styles/admin.css' +import { formatDateTime } from '../lib/date-utils' export default function AdminErrorLogs() { const queryClient = useQueryClient() @@ -161,7 +162,7 @@ export default function AdminErrorLogs() { } const formatDate = (dateString: string) => { - return new Date(dateString).toLocaleString() + return formatDateTime(dateString) } if (listStatus === 'error') { diff --git a/packages/web/src/pages/AdminUsers.tsx b/packages/web/src/pages/AdminUsers.tsx index 47282c0c..f019390b 100644 --- a/packages/web/src/pages/AdminUsers.tsx +++ b/packages/web/src/pages/AdminUsers.tsx @@ -7,6 +7,7 @@ import { useFocusSearchOnSlash } from '../hooks/useHotkey' import { AsyncPresentation, Modal, PageError, fromQuery, getQueryErrorMessage } from '../ui' import '../styles/admin.css' import { EMPTY_LIST } from '../lib/empty-list' +import { parseDisplayDate } from '../lib/date-utils' interface CreateUserData { email: string @@ -402,12 +403,12 @@ export default function AdminUsers() { {user.createdAt - ? new Date(user.createdAt).toLocaleDateString() + ? parseDisplayDate(user.createdAt).toLocaleDateString() : 'N/A'} {user.lastLogin - ? new Date(user.lastLogin).toLocaleDateString() + ? parseDisplayDate(user.lastLogin).toLocaleDateString() : 'Never'} diff --git a/packages/web/src/pages/ContainerDetail.tsx b/packages/web/src/pages/ContainerDetail.tsx index bbde5b05..9bd7ceb2 100644 --- a/packages/web/src/pages/ContainerDetail.tsx +++ b/packages/web/src/pages/ContainerDetail.tsx @@ -23,7 +23,7 @@ import { hasContainerDisplayIdentifier, } from '../lib/container-display' import '../styles/storage.css' -import { formatDate } from '../lib/date-utils' +import { formatDate, parseDisplayDate } from '../lib/date-utils' function statusColor(name: string): string { const key = name.toLowerCase() @@ -453,11 +453,11 @@ export default function ContainerDetail() {

{container.created && ( - Created {new Date(container.created).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })} + Created {parseDisplayDate(container.created).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })} )} {container.created && container.lastUpdated && ' · '} {container.lastUpdated && ( - Updated {new Date(container.lastUpdated).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })} + Updated {parseDisplayDate(container.lastUpdated).toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' })} )}

diff --git a/packages/web/src/pages/LocationDetail.tsx b/packages/web/src/pages/LocationDetail.tsx index c5631f70..95418dd0 100644 --- a/packages/web/src/pages/LocationDetail.tsx +++ b/packages/web/src/pages/LocationDetail.tsx @@ -13,6 +13,7 @@ import { DetailPageSkeleton, PageError, fromQuery, getQueryErrorMessage } from ' import ContentCard from '../components/ContentCard' import '../styles/storage.css' import { EMPTY_LIST } from '../lib/empty-list' +import { parseDisplayDate } from '../lib/date-utils' interface LocationContents { micronixPlates?: any[] @@ -177,14 +178,14 @@ export default function LocationDetail() {

Created

- {new Date(location.created).toLocaleDateString()} + {parseDisplayDate(location.created).toLocaleDateString()}

Last Updated

- {new Date(location.lastUpdated).toLocaleDateString()} + {parseDisplayDate(location.lastUpdated).toLocaleDateString()}

diff --git a/packages/web/src/pages/Locations.tsx b/packages/web/src/pages/Locations.tsx index 95a19bc1..b2e402e4 100644 --- a/packages/web/src/pages/Locations.tsx +++ b/packages/web/src/pages/Locations.tsx @@ -22,6 +22,7 @@ import { useUser } from '../contexts/UserContext' import { useFocusSearchOnSlash } from '../hooks/useHotkey' import { useClickOutside } from '../hooks/useClickOutside' import '../styles/storage.css' +import { parseDisplayDate } from '../lib/date-utils' interface Location { id: number @@ -663,9 +664,9 @@ export default function Locations() { )}

Created{' '} - {new Date(location.created).toLocaleDateString()} • Last + {parseDisplayDate(location.created).toLocaleDateString()} • Last updated{' '} - {new Date(location.lastUpdated).toLocaleDateString()} + {parseDisplayDate(location.lastUpdated).toLocaleDateString()}

diff --git a/packages/web/src/pages/SpecimenDetail.tsx b/packages/web/src/pages/SpecimenDetail.tsx index 6560ef00..da56f1cd 100644 --- a/packages/web/src/pages/SpecimenDetail.tsx +++ b/packages/web/src/pages/SpecimenDetail.tsx @@ -21,7 +21,7 @@ import { getQueryErrorMessage, } from '../ui' import '../styles/subject-specimen.css' -import { formatDate } from '../lib/date-utils' +import { formatDate, parseDisplayDate } from '../lib/date-utils' interface Container { id: number @@ -228,7 +228,7 @@ export default function SpecimenDetail() {
Created
- {new Date(specimen.created).toLocaleDateString()} + {parseDisplayDate(specimen.created).toLocaleDateString()}
diff --git a/packages/web/src/pages/Statistics.tsx b/packages/web/src/pages/Statistics.tsx index 1079aa25..0c8dd046 100644 --- a/packages/web/src/pages/Statistics.tsx +++ b/packages/web/src/pages/Statistics.tsx @@ -8,7 +8,7 @@ import { SkeletonCard } from '../ui' import { useStatistics } from '../hooks/useStatistics' import { PageError, fromQuery, getQueryErrorMessage } from '../ui' import '../styles/statistics.css' -import { toLocalDateString } from '../lib/date-utils' +import { toLocalDateString, parseDisplayDate, displayDateTime } from '../lib/date-utils' type BinSize = 'day' | 'week' | 'month' | 'quarter' | 'year' @@ -173,7 +173,7 @@ export default function Statistics() { // Helper function to get bin key for a date based on bin size const getBinKey = (dateString: string, size: BinSize): string => { - const date = new Date(dateString) + const date = parseDisplayDate(dateString) const year = date.getFullYear() const month = date.getMonth() + 1 const day = date.getDate() @@ -206,7 +206,7 @@ export default function Statistics() { const formatBinKey = (key: string, size: BinSize): string => { switch (size) { case 'day': { - const date = new Date(key) + const date = parseDisplayDate(key) return date.toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' }) } case 'week': { @@ -229,10 +229,10 @@ export default function Statistics() { const collectionTimelineData = useMemo(() => { if (!data) return [] - const minTimestamp = histogramMinDate ? new Date(histogramMinDate).getTime() : 0 + const minTimestamp = histogramMinDate ? displayDateTime(histogramMinDate) : 0 const grouped = data.specimens.collectionTimeline .filter(({ date }) => { - const dateTimestamp = new Date(date).getTime() + const dateTimestamp = displayDateTime(date) return dateTimestamp >= minTimestamp }) .reduce((acc, { date, count }) => { @@ -255,10 +255,10 @@ export default function Statistics() { const creationTimelineData = useMemo(() => { if (!data) return [] - const minTimestamp = histogramMinDate ? new Date(histogramMinDate).getTime() : 0 + const minTimestamp = histogramMinDate ? displayDateTime(histogramMinDate) : 0 const grouped = data.specimens.creationTimeline .filter(({ date }) => { - const dateTimestamp = new Date(date).getTime() + const dateTimestamp = displayDateTime(date) return dateTimestamp >= minTimestamp }) .reduce((acc, { date, count }) => { diff --git a/packages/web/src/pages/Studies.tsx b/packages/web/src/pages/Studies.tsx index adb66246..5dabcbb2 100644 --- a/packages/web/src/pages/Studies.tsx +++ b/packages/web/src/pages/Studies.tsx @@ -21,6 +21,7 @@ import { getQueryErrorMessage, } from '../ui' import '../styles/studies.css' +import { displayDateTime } from '../lib/date-utils' type ViewMode = 'grid' | 'list' type SortOption = 'title' | 'date' | 'subjects' | 'specimens' | 'containers' | 'lead' @@ -222,7 +223,7 @@ export default function Studies() { result = a.title.localeCompare(b.title) break case 'date': - result = new Date(b.lastUpdated).getTime() - new Date(a.lastUpdated).getTime() + result = displayDateTime(b.lastUpdated) - displayDateTime(a.lastUpdated) break case 'lead': result = a.leadPerson.localeCompare(b.leadPerson) diff --git a/packages/web/src/pages/StudyDetail.tsx b/packages/web/src/pages/StudyDetail.tsx index 3cf496f0..8b979813 100644 --- a/packages/web/src/pages/StudyDetail.tsx +++ b/packages/web/src/pages/StudyDetail.tsx @@ -35,6 +35,7 @@ import { } from '../ui' import { TUTORIAL_SHORT_CODE_PREFIX } from '../lib/constants' import '../styles/studies.css' +import { parseDisplayDate } from '../lib/date-utils' export default function StudyDetail() { const { id } = useParams<{ id: string }>() @@ -251,13 +252,13 @@ export default function StudyDetail() { key: 'created', label: 'Created', sortable: true, - render: (value) => new Date(value as string).toLocaleDateString(), + render: (value) => parseDisplayDate(value as string).toLocaleDateString(), }, { key: 'lastUpdated', label: 'Last Updated', sortable: true, - render: (value) => new Date(value as string).toLocaleDateString(), + render: (value) => parseDisplayDate(value as string).toLocaleDateString(), }, ] From e81227a0fc2b14ca40ff788d5d02e9168123a536 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:39:41 -0700 Subject: [PATCH 10/54] refactor(position): one Grid position rule for reads and writes #102 and #103 each added a contract module with its own geometry and its own position regex. Fold collection-grid.ts into grid-position.ts: GRID_GEOMETRY is the only geometry, parseGridPosition the only parser, and gridPositionKey is parseGridPosition with failure meaning "unplaced", so a position a write would reject can never be placed by a read. - gridAxes(type), gridCellKeys(type) and gridPositionKey(position, type) take the Collection type; MICRONIX_PLATE_GRID, CRYOVIAL_BOX_GRID and CollectionGrid are gone and every caller is updated. - Columns take one or two digits (#102's rule). No geometry has more than 12 columns, and no write path produces "A001"; #103's 1-3 digit read pattern would have placed positions that writes reject. - isStoredGridPosition says whether a stored value is already in the form the rule produces, for the integrity report. - container-move and specimen-registration use the contract's GridCollectionType instead of their own copies. - CONTEXT.md: Grid position defines the rule once; Unplaced container refers to it. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 4 +- .../src/lib/collections/collection-detail.ts | 6 +- packages/api/src/lib/container-move.ts | 5 +- packages/api/src/lib/plate-scan-validation.ts | 8 +- .../src/lib/specimen-registration/tables.ts | 3 +- .../src/__tests__/collection-grid.test.ts | 54 ---------- .../src/__tests__/grid-position.test.ts | 98 ++++++++++++++++++- packages/contract/src/collection-grid.ts | 53 ---------- packages/contract/src/grid-position.ts | 68 ++++++++++--- packages/contract/src/index.ts | 13 +-- .../web/src/lib/micronix-plate-positions.ts | 13 +-- packages/web/src/pages/CryovialBoxDetail.tsx | 10 +- .../web/src/pages/MicronixPlateDetail.tsx | 10 +- 13 files changed, 184 insertions(+), 161 deletions(-) delete mode 100644 packages/contract/src/__tests__/collection-grid.test.ts delete mode 100644 packages/contract/src/collection-grid.ts diff --git a/CONTEXT.md b/CONTEXT.md index 133330b5..6b3d4d32 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -132,11 +132,11 @@ Where a **Container** sits in the physical storage hierarchy — its **grid posi _Avoid_: Enrichment (implementation term); conflating placement with specimen or source lookups; placing paper directly on a box or bag without a sheet **Grid position**: -The well or slot of a tube or well **Container** in its plate or box (e.g. A01): a row letter plus a two-digit column, inside the **Collection**'s geometry (micronix plate A01–H12, cryovial box A01–J10). Every write that sets one normalizes `a1` to `A01` and rejects anything else. Legacy rows may still hold malformed positions. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. +The well or slot of a tube or well **Container** in its plate or box, stored as a row letter plus a two-digit column (A01). One rule, the Grid position rule, says which positions a **Collection** has (micronix plate A01–H12, cryovial box A01–J10) and how a typed position maps onto one: `a1`, `A1` and `a01` all mean A01, and anything else, including a cell outside the plate or box, is not a Grid position. Every write that sets a position applies the rule and rejects a position it does not accept; every read that lays Containers out on the grid applies the same rule, so a position a write would reject places no Container (see **Unplaced container**). Legacy rows may still hold malformed positions. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. _Avoid_: Well (as the domain term); treating a missing position as occupying a slot; requiring every tube to have a position before it can exist in a Collection **Unplaced container**: -A tube or well **Container** that belongs to a plate or box but occupies no cell: it has no **Grid position**, or its stored position is not a cell of the Collection's grid (rows A–H, columns 1–12 on a micronix plate; rows A–J, columns 1–10 on a cryovial box). It still counts as part of the Collection. Plate and box pages list unplaced Containers in their own section, labelled with the stored value, and plate scan validation expects them rather than treating them as extras. +A tube or well **Container** that belongs to a plate or box but occupies no cell: it has no **Grid position**, or its stored position fails the Grid position rule for that Collection. It still counts as part of the Collection. Plate and box pages list unplaced Containers in their own section, labelled with the stored value, and plate scan validation expects them rather than treating them as extras. _Avoid_: Empty (an empty cell holds no Container); treating an unplaced Container as missing from its Collection ### Export diff --git a/packages/api/src/lib/collections/collection-detail.ts b/packages/api/src/lib/collections/collection-detail.ts index 38cde873..330c7164 100644 --- a/packages/api/src/lib/collections/collection-detail.ts +++ b/packages/api/src/lib/collections/collection-detail.ts @@ -12,7 +12,7 @@ import { sheet, location, } from '../../db/schema' -import { CRYOVIAL_BOX_GRID, MICRONIX_PLATE_GRID, gridPositionKey } from '@sampledb/contract' +import { gridPositionKey } from '@sampledb/contract' import { formatLocationPath } from '../container-enrichment' import { enrichPaperContainers, enrichStorageContainers } from './container-detail' @@ -69,7 +69,7 @@ export async function getMicronixPlateDetail(database: Database, id: number) { const wellsByPosition: Record = {} const unplaced: PlateEntry[] = [] for (const entry of [...tubeEntries, ...wellEntries]) { - const key = gridPositionKey(entry.position, MICRONIX_PLATE_GRID) + const key = gridPositionKey(entry.position, 'micronix_plate') if (key != null && !(key in wellsByPosition)) wellsByPosition[key] = entry else unplaced.push(entry) } @@ -104,7 +104,7 @@ export async function getCryovialBoxDetail(database: Database, id: number) { const positions: Record = {} const unplaced: typeof tubeEntries = [] for (const entry of tubeEntries) { - const key = gridPositionKey(entry.position, CRYOVIAL_BOX_GRID) + const key = gridPositionKey(entry.position, 'cryovial_box') if (key == null) unplaced.push(entry) else (positions[key] ??= []).push(entry) } diff --git a/packages/api/src/lib/container-move.ts b/packages/api/src/lib/container-move.ts index 2e786cc2..7fb0d926 100644 --- a/packages/api/src/lib/container-move.ts +++ b/packages/api/src/lib/container-move.ts @@ -14,7 +14,7 @@ import { eq, and, isNull } from 'drizzle-orm' import { resolveCollection, resolveSheetsByName, type CollectionType } from './collections/collection-resolve' import { checkGridPositionOccupancy } from './container-occupancy' import { normalizePosition } from './normalize-position' -import { parseGridPosition } from '@sampledb/contract' +import { parseGridPosition, type GridCollectionType } from '@sampledb/contract' import type { ContainerSubtype } from './container-placement' /** Staging prefix for position updates so swaps on one plate do not violate UNIQUE(collection_id, position) mid-transaction. */ @@ -68,9 +68,6 @@ export interface MoveResult { errors?: ValidationError[] } -/** Collections whose containers sit at grid positions. */ -type GridCollectionType = 'cryovial_box' | 'micronix_plate' - /** * Resolve container by position and collection */ diff --git a/packages/api/src/lib/plate-scan-validation.ts b/packages/api/src/lib/plate-scan-validation.ts index 19d257b1..5f8aefbd 100644 --- a/packages/api/src/lib/plate-scan-validation.ts +++ b/packages/api/src/lib/plate-scan-validation.ts @@ -8,7 +8,7 @@ import { tag, } from '../db/schema' import { eq, inArray } from 'drizzle-orm' -import { MICRONIX_PLATE_GRID, gridCellKeys, gridPositionKey } from '@sampledb/contract' +import { gridCellKeys, gridPositionKey } from '@sampledb/contract' import { getScannerConfigurationById } from './settings' import { parsePlateCSV, normalizeWellPosition } from './plate-csv' @@ -278,7 +278,7 @@ export async function validatePlateScan( const unplacedByBarcode = new Map & { barcode: string; storedPosition: string | null; containerId: number }>() for (const t of tubes) { - const pos = gridPositionKey(t.position, MICRONIX_PLATE_GRID) + const pos = gridPositionKey(t.position, 'micronix_plate') if (pos != null && !expectedByPosition.has(pos)) { expectedByPosition.set(pos, { position: pos, containerId: t.id, barcode: t.barcode, remainingQuantity: null, tags: [] }) } else { @@ -292,7 +292,7 @@ export async function validatePlateScan( } } for (const w of wells) { - const pos = gridPositionKey(w.position, MICRONIX_PLATE_GRID) + const pos = gridPositionKey(w.position, 'micronix_plate') if (pos != null && !expectedByPosition.has(pos)) { expectedByPosition.set(pos, { position: pos, containerId: w.id, barcode: null, remainingQuantity: null, tags: [] }) } @@ -336,7 +336,7 @@ export async function validatePlateScan( if (barcode.trim() !== '') allPositions.add(pos) else if (expectedByPosition.has(pos)) allPositions.add(pos) } - const sortedPositions = gridCellKeys(MICRONIX_PLATE_GRID).filter((p) => allPositions.has(p)) + const sortedPositions = gridCellKeys('micronix_plate').filter((p) => allPositions.has(p)) const rest = [...allPositions].filter((p) => !sortedPositions.includes(p)).sort() const positionOrder = [...sortedPositions, ...rest] diff --git a/packages/api/src/lib/specimen-registration/tables.ts b/packages/api/src/lib/specimen-registration/tables.ts index 2cec3789..38eff864 100644 --- a/packages/api/src/lib/specimen-registration/tables.ts +++ b/packages/api/src/lib/specimen-registration/tables.ts @@ -1,8 +1,9 @@ /** Which Collection each container type goes in, and the table behind each type. One place. */ +import type { GridCollectionType } from '@sampledb/contract' import { bag, box, cryovialBox, cryovialTube, micronixPlate, micronixTube, staticWell } from '../../db/schema' export type GridContainerType = 'micronix_tube' | 'cryovial_tube' | 'static_well' -export type GridCollectionType = 'micronix_plate' | 'cryovial_box' +export type { GridCollectionType } export type ParentCollectionType = 'box' | 'bag' /** The plate or box a tube or well goes in. */ diff --git a/packages/contract/src/__tests__/collection-grid.test.ts b/packages/contract/src/__tests__/collection-grid.test.ts deleted file mode 100644 index 42e554f7..00000000 --- a/packages/contract/src/__tests__/collection-grid.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { describe, expect, it } from 'bun:test' -import { - CRYOVIAL_BOX_GRID, - MICRONIX_PLATE_GRID, - gridCellKeys, - gridPositionKey, -} from '../collection-grid' - -describe('collection grid geometry', () => { - it('orders cryovial box rows and columns naturally', () => { - expect(CRYOVIAL_BOX_GRID.rows).toEqual(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']) - expect(CRYOVIAL_BOX_GRID.columns).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]) - }) - - it('lists every cell of a micronix plate row by row', () => { - const keys = gridCellKeys(MICRONIX_PLATE_GRID) - expect(keys).toHaveLength(96) - expect(keys.slice(0, 3)).toEqual(['A01', 'A02', 'A03']) - expect(keys[12]).toBe('B01') - expect(keys.at(-1)).toBe('H12') - }) -}) - -describe('gridPositionKey', () => { - it('normalizes a valid position to its cell key', () => { - expect(gridPositionKey('A01', MICRONIX_PLATE_GRID)).toBe('A01') - expect(gridPositionKey('a1', MICRONIX_PLATE_GRID)).toBe('A01') - expect(gridPositionKey(' H12 ', MICRONIX_PLATE_GRID)).toBe('H12') - expect(gridPositionKey('J10', CRYOVIAL_BOX_GRID)).toBe('J10') - }) - - it('returns null for a missing position', () => { - expect(gridPositionKey(null, MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey(undefined, MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey('', MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey(' ', MICRONIX_PLATE_GRID)).toBeNull() - }) - - it('returns null for a position outside the grid', () => { - expect(gridPositionKey('I01', MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey('A13', MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey('A00', MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey('K01', CRYOVIAL_BOX_GRID)).toBeNull() - expect(gridPositionKey('A11', CRYOVIAL_BOX_GRID)).toBeNull() - expect(gridPositionKey('Z99', CRYOVIAL_BOX_GRID)).toBeNull() - }) - - it('returns null for stored values that are not a row letter plus column number', () => { - expect(gridPositionKey('--A01', CRYOVIAL_BOX_GRID)).toBeNull() - expect(gridPositionKey('NA01', CRYOVIAL_BOX_GRID)).toBeNull() - expect(gridPositionKey('A1145868849', MICRONIX_PLATE_GRID)).toBeNull() - expect(gridPositionKey('hello world', MICRONIX_PLATE_GRID)).toBeNull() - }) -}) diff --git a/packages/contract/src/__tests__/grid-position.test.ts b/packages/contract/src/__tests__/grid-position.test.ts index c7ef5722..6208d561 100644 --- a/packages/contract/src/__tests__/grid-position.test.ts +++ b/packages/contract/src/__tests__/grid-position.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from 'bun:test' -import { gridPositionRange, parseGridPosition, type GridCollectionType } from '../grid-position' +import { + GRID_GEOMETRY, + gridAxes, + gridCellKeys, + gridPositionKey, + gridPositionRange, + isStoredGridPosition, + parseGridPosition, + type GridCollectionType, +} from '../grid-position' describe('parseGridPosition', () => { const normalized: Array<{ raw: string; type: GridCollectionType; expected: string }> = [ @@ -55,3 +64,90 @@ describe('parseGridPosition', () => { expect(gridPositionRange('cryovial_box')).toBe('A01–J10') }) }) + +describe('grid geometry', () => { + it('orders cryovial box rows and columns naturally', () => { + expect(gridAxes('cryovial_box')).toEqual({ + rows: ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'], + columns: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + }) + }) + + it('lists every cell of a micronix plate row by row', () => { + const keys = gridCellKeys('micronix_plate') + expect(keys).toHaveLength(96) + expect(keys.slice(0, 3)).toEqual(['A01', 'A02', 'A03']) + expect(keys[12]).toBe('B01') + expect(keys.at(-1)).toBe('H12') + }) + + it('accepts every cell it lists, and nothing past the last row or column', () => { + for (const type of Object.keys(GRID_GEOMETRY) as GridCollectionType[]) { + for (const key of gridCellKeys(type)) { + expect(parseGridPosition(key, type)).toEqual({ ok: true, position: key }) + } + const { rows, columns } = GRID_GEOMETRY[type] + expect(parseGridPosition(`A${columns + 1}`, type).ok).toBe(false) + expect(parseGridPosition(`${String.fromCharCode(65 + rows)}01`, type).ok).toBe(false) + } + }) +}) + +describe('gridPositionKey', () => { + it('normalizes a valid position to its cell key', () => { + expect(gridPositionKey('A01', 'micronix_plate')).toBe('A01') + expect(gridPositionKey('a1', 'micronix_plate')).toBe('A01') + expect(gridPositionKey(' H12 ', 'micronix_plate')).toBe('H12') + expect(gridPositionKey('J10', 'cryovial_box')).toBe('J10') + }) + + it('returns null for a missing position', () => { + expect(gridPositionKey(null, 'micronix_plate')).toBeNull() + expect(gridPositionKey(undefined, 'micronix_plate')).toBeNull() + expect(gridPositionKey('', 'micronix_plate')).toBeNull() + expect(gridPositionKey(' ', 'micronix_plate')).toBeNull() + }) + + it('returns null for a position outside the grid', () => { + expect(gridPositionKey('I01', 'micronix_plate')).toBeNull() + expect(gridPositionKey('A13', 'micronix_plate')).toBeNull() + expect(gridPositionKey('A00', 'micronix_plate')).toBeNull() + expect(gridPositionKey('K01', 'cryovial_box')).toBeNull() + expect(gridPositionKey('A11', 'cryovial_box')).toBeNull() + expect(gridPositionKey('Z99', 'cryovial_box')).toBeNull() + }) + + it('returns null for stored values that are not a row letter plus column number', () => { + expect(gridPositionKey('--A01', 'cryovial_box')).toBeNull() + expect(gridPositionKey('NA01', 'cryovial_box')).toBeNull() + expect(gridPositionKey('A001', 'cryovial_box')).toBeNull() + expect(gridPositionKey('A1145868849', 'micronix_plate')).toBeNull() + expect(gridPositionKey('hello world', 'micronix_plate')).toBeNull() + }) + + it('places exactly the positions a write accepts', () => { + const samples = ['A1', 'a01', 'B012', 'A001', '--A01', 'NA01', 'H12', 'I01', 'J10', 'K01', ' c3 ', ''] + for (const type of Object.keys(GRID_GEOMETRY) as GridCollectionType[]) { + for (const raw of samples) { + const parsed = parseGridPosition(raw, type) + expect(gridPositionKey(raw, type)).toBe(parsed.ok ? parsed.position : null) + } + } + }) +}) + +describe('isStoredGridPosition', () => { + it('accepts a missing position and a position in stored form', () => { + expect(isStoredGridPosition(null, 'cryovial_box')).toBe(true) + expect(isStoredGridPosition('J10', 'cryovial_box')).toBe(true) + expect(isStoredGridPosition('H12', 'micronix_plate')).toBe(true) + }) + + it('rejects a position the rule rejects or would store differently', () => { + expect(isStoredGridPosition('--A01', 'cryovial_box')).toBe(false) + expect(isStoredGridPosition('NA01', 'cryovial_box')).toBe(false) + expect(isStoredGridPosition('A1', 'cryovial_box')).toBe(false) + expect(isStoredGridPosition('a01', 'micronix_plate')).toBe(false) + expect(isStoredGridPosition('J10', 'micronix_plate')).toBe(false) + }) +}) diff --git a/packages/contract/src/collection-grid.ts b/packages/contract/src/collection-grid.ts deleted file mode 100644 index e2da2b4d..00000000 --- a/packages/contract/src/collection-grid.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Grid geometry of the Collections that hold tube and well Containers, and the - * rule that maps a stored grid position onto that geometry. - * - * A stored position that does not map onto the grid (missing, out of range, or - * not a row letter plus column number) leaves its Container unplaced: it - * belongs to the Collection but occupies no cell. - */ - -export type CollectionGrid = { - /** Row letters, top to bottom. */ - readonly rows: readonly string[] - /** Column numbers, left to right. */ - readonly columns: readonly number[] -} - -function letters(count: number): string[] { - return Array.from({ length: count }, (_, i) => String.fromCharCode(65 + i)) -} - -function numbers(count: number): number[] { - return Array.from({ length: count }, (_, i) => i + 1) -} - -/** 96-well micronix plate: rows A–H, columns 1–12. */ -export const MICRONIX_PLATE_GRID: CollectionGrid = { rows: letters(8), columns: numbers(12) } - -/** 100-slot cryovial box: rows A–J, columns 1–10. */ -export const CRYOVIAL_BOX_GRID: CollectionGrid = { rows: letters(10), columns: numbers(10) } - -/** Cell key for a row and column, e.g. `B` and `7` give `B07`. */ -export function gridCellKey(row: string, column: number): string { - return `${row}${String(column).padStart(2, '0')}` -} - -/** Every cell key of the grid, row by row (A01, A02, …, B01, …). */ -export function gridCellKeys(grid: CollectionGrid): string[] { - return grid.rows.flatMap((row) => grid.columns.map((column) => gridCellKey(row, column))) -} - -/** - * The grid cell a stored position names, or null when the Container is unplaced. - * `a1`, `A1` and ` A01 ` all give `A01`; `Z99`, `--A01`, `NA01` and null give null. - */ -export function gridPositionKey(position: string | null | undefined, grid: CollectionGrid): string | null { - if (position == null) return null - const match = /^([A-Za-z])(\d{1,3})$/.exec(position.trim()) - if (!match) return null - const row = match[1]!.toUpperCase() - const column = parseInt(match[2]!, 10) - if (!grid.rows.includes(row) || !grid.columns.includes(column)) return null - return gridCellKey(row, column) -} diff --git a/packages/contract/src/grid-position.ts b/packages/contract/src/grid-position.ts index 9ba443c3..c303eacd 100644 --- a/packages/contract/src/grid-position.ts +++ b/packages/contract/src/grid-position.ts @@ -1,7 +1,13 @@ /** - * The Grid position rule: which wells or slots a plate or box has, and how a typed position - * maps onto one. Every Container write that sets a Grid position (moves, Scan moves, bulk - * import, Specimen registration, derivations) checks the position with `parseGridPosition`. + * The Grid position rule: which wells or slots a plate or box has, and how a position maps + * onto one. It is the only geometry and the only parser for Grid positions, so the write + * paths and the read paths cannot disagree: + * + * - Every Container write that sets a Grid position (moves, Scan moves, bulk import, Specimen + * registration, derivations) checks the position with `parseGridPosition`. + * - Every read that lays Containers out on the grid (plate and box detail, plate scan + * validation) finds a Container's cell with `gridPositionKey`, which is `parseGridPosition` + * with failure meaning "unplaced". */ /** A Collection whose Containers sit at Grid positions. */ @@ -31,16 +37,36 @@ const COLLECTION_LABEL: Record = { cryovial_box: 'cryovial box', } +/** + * A row letter and a column number of one or two digits. No geometry has more than 12 + * columns, so a third digit (`A001`) is never needed, and no write path produces it. + */ const POSITION_PATTERN = /^([A-Za-z])(\d{1,2})$/ -/** The range of a Collection type's Grid positions, as in "A01–H12". */ -export function gridPositionRange(collectionType: GridCollectionType): string { +/** Row letters and column numbers of a Collection type's grid, in display order. */ +export function gridAxes(collectionType: GridCollectionType): { rows: string[]; columns: number[] } { const { rows, columns } = GRID_GEOMETRY[collectionType] - return `A01–${formatGridPosition(rows - 1, columns)}` + return { + rows: Array.from({ length: rows }, (_, i) => String.fromCharCode(65 + i)), + columns: Array.from({ length: columns }, (_, i) => i + 1), + } } -function formatGridPosition(rowIndex: number, column: number): string { - return `${String.fromCharCode(65 + rowIndex)}${String(column).padStart(2, '0')}` +/** Stored form of a row and column, e.g. `B` and `7` give `B07`. */ +export function gridCellKey(row: string, column: number): string { + return `${row}${String(column).padStart(2, '0')}` +} + +/** Every stored Grid position of a Collection type, row by row (A01, A02, …, B01, …). */ +export function gridCellKeys(collectionType: GridCollectionType): string[] { + const { rows, columns } = gridAxes(collectionType) + return rows.flatMap((row) => columns.map((column) => gridCellKey(row, column))) +} + +/** The range of a Collection type's Grid positions, as in "A01–H12". */ +export function gridPositionRange(collectionType: GridCollectionType): string { + const { rows, columns } = GRID_GEOMETRY[collectionType] + return `A01–${gridCellKey(String.fromCharCode(64 + rows), columns)}` } /** @@ -63,11 +89,31 @@ export function parseGridPosition(raw: string, collectionType: GridCollectionTyp } } - const rowIndex = match[1]!.toUpperCase().charCodeAt(0) - 65 + const row = match[1]!.toUpperCase() const column = parseInt(match[2]!, 10) const { rows, columns } = GRID_GEOMETRY[collectionType] - if (rowIndex >= rows || column < 1 || column > columns) { + if (row.charCodeAt(0) - 65 >= rows || column < 1 || column > columns) { return { ok: false, error: `Position "${value}" is outside a ${label} (${range})` } } - return { ok: true, position: formatGridPosition(rowIndex, column) } + return { ok: true, position: gridCellKey(row, column) } +} + +/** + * The grid cell a stored position names, or null when the Container is unplaced: its position + * is missing or fails `parseGridPosition`. `a1`, `A1` and ` A01 ` all give `A01`; `Z99`, + * `--A01`, `NA01` and null give null. + */ +export function gridPositionKey(position: string | null | undefined, collectionType: GridCollectionType): string | null { + if (position == null) return null + const parsed = parseGridPosition(position, collectionType) + return parsed.ok ? parsed.position : null +} + +/** + * Whether a stored position is valid as stored: missing, or already in the stored form the + * rule produces. Occupancy compares stored strings, so `A1` next to `A01` would let two + * Containers share a cell; the integrity report flags any position this rejects. + */ +export function isStoredGridPosition(position: string | null | undefined, collectionType: GridCollectionType): boolean { + return position == null || gridPositionKey(position, collectionType) === position } diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1a3add2f..66571077 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -83,17 +83,14 @@ export { } from './paper-container-inbound' export { GRID_GEOMETRY, + gridAxes, + gridCellKey, + gridCellKeys, + gridPositionKey, gridPositionRange, + isStoredGridPosition, parseGridPosition, type GridCollectionType, type GridGeometry, type GridPositionResult, } from './grid-position' -export { - CRYOVIAL_BOX_GRID, - MICRONIX_PLATE_GRID, - gridCellKey, - gridCellKeys, - gridPositionKey, - type CollectionGrid, -} from './collection-grid' diff --git a/packages/web/src/lib/micronix-plate-positions.ts b/packages/web/src/lib/micronix-plate-positions.ts index feb9ca53..834d1232 100644 --- a/packages/web/src/lib/micronix-plate-positions.ts +++ b/packages/web/src/lib/micronix-plate-positions.ts @@ -2,24 +2,17 @@ * Well position utilities for micronix plates (96-well: A01–H12), on the shared Grid position * rule from `@sampledb/contract`. */ -import { GRID_GEOMETRY, parseGridPosition } from '@sampledb/contract' - -const { rows: PLATE_ROWS, columns: PLATE_COLUMNS } = GRID_GEOMETRY.micronix_plate +import { gridCellKeys, gridPositionKey } from '@sampledb/contract' /** Set of all 96 well positions for a standard micronix plate (8×12). */ -export const MICRONIX_WELL_POSITIONS_96: Set = new Set( - Array.from({ length: PLATE_ROWS }, (_, r) => - Array.from({ length: PLATE_COLUMNS }, (_, c) => `${String.fromCharCode(65 + r)}${String(c + 1).padStart(2, '0')}`), - ).flat(), -) +export const MICRONIX_WELL_POSITIONS_96: Set = new Set(gridCellKeys('micronix_plate')) /** * Normalize well position to A01 style (row A–H, column 01–12). * Returns normalized position or null if invalid. */ export function normalizeWellPosition(pos: string): string | null { - const parsed = parseGridPosition(pos, 'micronix_plate') - return parsed.ok ? parsed.position : null + return gridPositionKey(pos, 'micronix_plate') } export interface FullPlateValidationResult { diff --git a/packages/web/src/pages/CryovialBoxDetail.tsx b/packages/web/src/pages/CryovialBoxDetail.tsx index accb3c2c..7a0ec0f8 100644 --- a/packages/web/src/pages/CryovialBoxDetail.tsx +++ b/packages/web/src/pages/CryovialBoxDetail.tsx @@ -16,7 +16,7 @@ import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { Link } from 'react-router-dom' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' import UnplacedContainersSection from '../components/UnplacedContainersSection' -import { CRYOVIAL_BOX_GRID } from '@sampledb/contract' +import { gridAxes } from '@sampledb/contract' import '../styles/storage.css' function statusColor(name: string): string { @@ -63,10 +63,10 @@ export default function CryovialBoxDetail() { // The grid always shows the whole box (rows A–J, columns 1–10), whatever it holds. const layout = useMemo( - () => ({ - rows: [...CRYOVIAL_BOX_GRID.rows], - cols: CRYOVIAL_BOX_GRID.columns.map((column) => String(column)), - }), + () => { + const { rows, columns } = gridAxes('cryovial_box') + return { rows, cols: columns.map((column) => String(column)) } + }, [], ) diff --git a/packages/web/src/pages/MicronixPlateDetail.tsx b/packages/web/src/pages/MicronixPlateDetail.tsx index 23926fec..7e743341 100644 --- a/packages/web/src/pages/MicronixPlateDetail.tsx +++ b/packages/web/src/pages/MicronixPlateDetail.tsx @@ -16,7 +16,7 @@ import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { Link } from 'react-router-dom' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' import UnplacedContainersSection from '../components/UnplacedContainersSection' -import { MICRONIX_PLATE_GRID } from '@sampledb/contract' +import { gridAxes } from '@sampledb/contract' import '../styles/storage.css' function statusColor(name: string): string { @@ -62,10 +62,10 @@ export default function MicronixPlateDetail() { }, [targetPosition, data]) const layout = useMemo( - () => ({ - rows: [...MICRONIX_PLATE_GRID.rows], - cols: MICRONIX_PLATE_GRID.columns.map((column) => String(column)), - }), + () => { + const { rows, columns } = gridAxes('micronix_plate') + return { rows, cols: columns.map((column) => String(column)) } + }, [], ) From 53a905595adc95135d7355f6e7b7e9289ef87a48 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:41:24 -0700 Subject: [PATCH 11/54] fix(collections): show delete impact and refresh the app after a delete (#113) The delete dialog now runs the preflight when it opens and lists the Containers, Specimens and Subjects the delete removes before the user confirms. Subjects count as removed only when "Also remove study participants" is on; otherwise the dialog says how many would be left with no Specimens. Preflight blockers show up front and disable Delete. The preflight summary gains emptySubjectCount: Subjects whose every Specimen the delete removes. The delete goes through useDeleteCollectionWithContents, which invalidates every query with invalidateAfterBulkWrite so the Collections list, dashboard, study, Subject and Location data refresh. The deleted Collection's own queries are marked stale without a refetch, so the detail page navigates away instead of refetching into a 404. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/collection-delete-cascade.ts | 32 +++ .../src/routes/__tests__/collections.test.ts | 58 +++- packages/contract/src/bulk-combined.ts | 2 + .../docs/guides/workflows/containers.md | 6 +- .../src/components/CollectionDeleteDialog.tsx | 62 +++- .../__tests__/CollectionDeleteDialog.test.tsx | 264 ++++++++++++++++++ packages/web/src/hooks/useCollections.ts | 47 +++- packages/web/src/lib/query-client.ts | 19 +- 8 files changed, 478 insertions(+), 12 deletions(-) create mode 100644 packages/web/src/components/__tests__/CollectionDeleteDialog.test.tsx diff --git a/packages/api/src/lib/collection-delete-cascade.ts b/packages/api/src/lib/collection-delete-cascade.ts index 82127e73..637a5a94 100644 --- a/packages/api/src/lib/collection-delete-cascade.ts +++ b/packages/api/src/lib/collection-delete-cascade.ts @@ -174,6 +174,36 @@ export async function computeSpecimensFullyRemovedByContainerBatch( return toRemove } +/** + * Subjects whose every Specimen is in `removedSpecimenIds`, so they would have no Specimens left after the + * delete. These are the Subjects `removeEmptySubjects` deletes. + */ +async function countSubjectsLeftWithoutSpecimens(database: Database, removedSpecimenIds: number[]): Promise { + if (removedSpecimenIds.length === 0) return 0 + const removedBySubject = new Map() + for (let i = 0; i < removedSpecimenIds.length; i += SQLITE_BATCH) { + const rows = await database + .select({ studySubjectId: specimen.studySubjectId }) + .from(specimen) + .where(inArray(specimen.id, removedSpecimenIds.slice(i, i + SQLITE_BATCH))) + .all() + for (const { studySubjectId } of rows) { + if (studySubjectId == null) continue + removedBySubject.set(studySubjectId, (removedBySubject.get(studySubjectId) ?? 0) + 1) + } + } + let count = 0 + for (const [subjectId, removed] of removedBySubject) { + const total = await database + .select({ n: sql`count(*)`.mapWith(Number) }) + .from(specimen) + .where(eq(specimen.studySubjectId, subjectId)) + .get() + if ((total?.n ?? 0) === removed) count++ + } + return count +} + function buildQpcrContainerBlockers( links: { wellId: number; experimentId: number; experimentName: string | null; wellPosition: string; storageId: number }[] ): CollectionDeleteBlocker { @@ -324,6 +354,7 @@ export async function preflightCollectionDelete( const containerIdSet = new Set(containerIdList) const fullyRemovedSpecimenIds = await computeSpecimensFullyRemovedByContainerBatch(database, containerIdSet) const blockers = await runPreflight(database, containerIdSet, fullyRemovedSpecimenIds) + const emptySubjectCount = await countSubjectsLeftWithoutSpecimens(database, fullyRemovedSpecimenIds) return { canDelete: blockers.length === 0, @@ -331,6 +362,7 @@ export async function preflightCollectionDelete( summary: { containerCount: containerIdList.length, specimenCount: fullyRemovedSpecimenIds.length, + emptySubjectCount, }, } } diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 7a80c107..14c95fcd 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -352,11 +352,65 @@ describe('Collections API', () => { const data = (await res.json()) as { canDelete: boolean blockers: Array<{ code: string; message: string }> - summary: { containerCount: number; specimenCount: number } + summary: { containerCount: number; specimenCount: number; emptySubjectCount: number } } expect(data.canDelete).toBe(true) expect(data.blockers).toEqual([]) - expect(data.summary).toEqual({ containerCount: 0, specimenCount: 0 }) + expect(data.summary).toEqual({ containerCount: 0, specimenCount: 0, emptySubjectCount: 0 }) + }) + + it('counts the Containers, Specimens and Subjects the delete would remove', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'StPreCount' }) + const loc = await createTestLocation(ctx.db, { + name: 'LocPreCount', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'PlatePreCount', locationId: loc.id }) + const otherPlate = await createTestMicronixPlate(ctx.db, { name: 'PlatePreOther', locationId: loc.id }) + const st = await createTestSpecimenType(ctx.db, { name: 'STPreCount' }) + const study = await createTestStudy(ctx.db, { title: 'Pre', shortCode: `PRE${Date.now()}` }) + const emptied = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Emptied' }) + const kept = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Kept' }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'm', category: 'volume' }) + const t = utcNow() + const addTube = async (specimenId: number, collectionId: number, position: string) => { + const [c] = await ctx.db + .insert(storageContainer) + .values({ specimenId, unitId: unit.id, totalQuantity: 1, remainingQuantity: 1, created: t, lastUpdated: t }) + .returning() + await ctx.db + .insert(micronixTube) + .values({ id: c!.id, collectionId, barcode: `BPRE${c!.id}`, position }) + } + // Emptied: two Specimens, both only on this plate. + const e1 = await createTestSpecimen(ctx.db, st.id, { studySubjectId: emptied.id }) + const e2 = await createTestSpecimen(ctx.db, st.id, { studySubjectId: emptied.id }) + await addTube(e1.id, plate.id, 'A01') + await addTube(e1.id, plate.id, 'A02') + await addTube(e2.id, plate.id, 'A03') + // Kept: one Specimen removed with the plate, one with an aliquot on another plate. + const k1 = await createTestSpecimen(ctx.db, st.id, { studySubjectId: kept.id }) + const k2 = await createTestSpecimen(ctx.db, st.id, { studySubjectId: kept.id }) + await addTube(k1.id, plate.id, 'A04') + await addTube(k2.id, plate.id, 'A05') + await addTube(k2.id, otherPlate.id, 'A01') + + const res = await ctx.request('/api/collections/delete-with-contents/preflight', { + method: 'POST', + json: { collectionType: 'micronix_plate', id: plate.id }, + }) + expect(res.status).toBe(200) + const data = (await res.json()) as { summary: unknown } + expect(data.summary).toEqual({ containerCount: 5, specimenCount: 3, emptySubjectCount: 1 }) + }) + + it('returns 404 when the collection does not exist', async () => { + const res = await ctx.request('/api/collections/delete-with-contents/preflight', { + method: 'POST', + json: { collectionType: 'micronix_plate', id: 999_999 }, + }) + expect(res.status).toBe(404) }) }) diff --git a/packages/contract/src/bulk-combined.ts b/packages/contract/src/bulk-combined.ts index bc97e09e..37b8efd6 100644 --- a/packages/contract/src/bulk-combined.ts +++ b/packages/contract/src/bulk-combined.ts @@ -182,6 +182,8 @@ export const collectionDeletePreflightSchema = z.object({ summary: z.object({ containerCount: z.number().int(), specimenCount: z.number().int(), + /** Subjects left with no Specimens after the delete; removed only when `removeEmptySubjects` is set. */ + emptySubjectCount: z.number().int(), }), }) diff --git a/packages/docs/src/content/docs/guides/workflows/containers.md b/packages/docs/src/content/docs/guides/workflows/containers.md index 33841342..40cdfc93 100644 --- a/packages/docs/src/content/docs/guides/workflows/containers.md +++ b/packages/docs/src/content/docs/guides/workflows/containers.md @@ -75,7 +75,9 @@ On a plate, cryovial box, box, or bag (not the sheet-only page), **Delete collec - Specimen records that would have no storage left (a specimen with another container elsewhere is kept) - Optional: with **Also remove study participants (subjects) that have no specimens left**, subjects who would have no specimens. Control batches are never removed this way. -Type the exact collection name to confirm. The operation is all or nothing. +Before you confirm, the dialog counts the containers, specimens, and subjects the delete removes. Subjects count only when the option to remove them is on; otherwise the dialog shows how many would be left with no specimens. Containers that belong to the collection but have no grid position are counted and removed too. + +Type the exact collection name to confirm. The operation is all or nothing. Afterwards the Collections list, dashboard, study and subject pages, and location contents show the result. ### When deletion is blocked @@ -85,6 +87,6 @@ Type the exact collection name to confirm. The operation is all or nothing. | `qpcr_wells_link_specimens` | A specimen you would remove is still on a qPCR well. | Clear the well, then try again. | | `container_derivation_spans_outside_collection` | A derivation parent is in this collection and the child is in another. | Move or remove the child, or reassign the derivation. Deleting a collection that only holds child containers is allowed. | -Nothing is partially deleted. Fix the listed codes and submit again. +The dialog lists blockers before you confirm, and keeps **Delete collection** disabled until they are fixed. Nothing is partially deleted. Fix the listed codes and open the dialog again. To move tubes between plates, see [Container movement](/docs/guides/bulk-operations/container-movement/). To move a whole plate or box to a new freezer, see [Move collections](/docs/guides/features/collection-move/). diff --git a/packages/web/src/components/CollectionDeleteDialog.tsx b/packages/web/src/components/CollectionDeleteDialog.tsx index e1165907..30d0c1ae 100644 --- a/packages/web/src/components/CollectionDeleteDialog.tsx +++ b/packages/web/src/components/CollectionDeleteDialog.tsx @@ -1,11 +1,20 @@ import { useState, useCallback } from 'react' import axios from 'axios' import type { CollectionDeletePreflight } from '@sampledb/contract' -import { collectionsApi } from '../lib/api/collections' +import { + useCollectionDeletePreflight, + useDeleteCollectionWithContents, + type DeletableCollectionType, +} from '../hooks/useCollections' +import { SectionMessage, getQueryErrorMessage } from '../ui' -type CollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' +type CollectionType = DeletableCollectionType type CollectionDeleteBlocker = CollectionDeletePreflight['blockers'][number] +function countLabel(n: number, singular: string, plural: string): string { + return `${n} ${n === 1 ? singular : plural}` +} + type Props = { isOpen: boolean onClose: () => void @@ -32,6 +41,8 @@ export default function CollectionDeleteDialog({ const [deleting, setDeleting] = useState(false) const [errorSummary, setErrorSummary] = useState(null) const [blockers, setBlockers] = useState([]) + const preflightQuery = useCollectionDeletePreflight(collectionType, id, isOpen) + const deleteMutation = useDeleteCollectionWithContents() const reset = useCallback(() => { setConfirmText('') @@ -49,6 +60,7 @@ export default function CollectionDeleteDialog({ }, [deleting, onClose, reset]) const nameMatches = confirmText.trim() === collectionName + const preflight = preflightQuery.data if (!isOpen) return null @@ -58,7 +70,7 @@ export default function CollectionDeleteDialog({ setErrorSummary(null) setBlockers([]) try { - await collectionsApi.deleteWithContents({ + await deleteMutation.mutateAsync({ collectionType, id, removeEmptySubjects, @@ -100,6 +112,48 @@ export default function CollectionDeleteDialog({ be undone. If a specimen still has other containers outside this collection, that specimen is kept. Use this when fixing a bad import or emptying a plate or box.

+
+ {preflightQuery.isPending ? ( + + ) : preflightQuery.isError ? ( + + ) : preflight ? ( + <> +

This removes:

+
    +
  • {countLabel(preflight.summary.containerCount, 'Container', 'Containers')}
    • +
  • {countLabel(preflight.summary.specimenCount, 'Specimen', 'Specimens')}
    • +
  • + {removeEmptySubjects + ? countLabel(preflight.summary.emptySubjectCount, 'Subject', 'Subjects') + : `0 Subjects (${countLabel(preflight.summary.emptySubjectCount, 'Subject', 'Subjects')} left with no Specimens)`} +
    • +
+ {!preflight.canDelete && ( +
+

This collection can't be deleted yet:

+
    + {preflight.blockers.map((b, i) => ( +
  • + {b.message} + + ({b.code}) + +
    • + ))} +
+
+ )} + + ) : null} +
@@ -560,9 +567,11 @@ export default function ScanMovePageShell({
{result.filename} - 0 ? 'text-app-trend-up' : 'text-app-text-muted'}`}> - {result.moved} moved - + {result.moved !== undefined && ( + 0 ? 'text-app-trend-up' : 'text-app-text-muted'}`}> + {result.moved} moved + + )}

Destination: {result.destinationName}

{result.errors && result.errors.length > 0 && ( @@ -585,7 +594,7 @@ export default function ScanMovePageShell({
    {moveResult.errors.map((error, i) => (
  • - {error.row > 0 ? Row {error.row}: : null}{' '} + {errorLocation(error) ? {errorLocation(error)}: : null}{' '} {error.error}
    • ))} diff --git a/packages/web/src/hooks/useScanMoveWorkflow.ts b/packages/web/src/hooks/useScanMoveWorkflow.ts index 7577e295..d0d805ff 100644 --- a/packages/web/src/hooks/useScanMoveWorkflow.ts +++ b/packages/web/src/hooks/useScanMoveWorkflow.ts @@ -90,15 +90,40 @@ export function useScanMoveWorkflow(options: UseScanMoveWorkflowOptions) { // eslint-disable-next-line react-hooks/exhaustive-deps -- mirror only on step change }, [state.step]) + // Work in flight; `loading` mirrors it for rendering. A ref, not state, + // so a second click in the same tick sees the first one. + const busyCountRef = useRef(0) + const actionInFlightRef = useRef(false) + const runBusy = useCallback(async (work: () => Promise) => { + busyCountRef.current++ setLoading(true) try { await work() } finally { - setLoading(false) + busyCountRef.current-- + setLoading(busyCountRef.current > 0) } }, []) + /** + * Run a resolve, create, or execute step at most once at a time. A repeat + * call while one is in flight (a double click) is dropped, so a second, + * failing move request can't overwrite the first one's result. + */ + const runExclusive = useCallback( + async (work: () => Promise) => { + if (actionInFlightRef.current) return + actionInFlightRef.current = true + try { + await runBusy(work) + } finally { + actionInFlightRef.current = false + } + }, + [runBusy], + ) + const ingestFiles = useCallback( (sources: ScanMoveFileSource[], scannerConfig?: ScannerConfiguration) => runBusy(async () => { @@ -176,17 +201,17 @@ export function useScanMoveWorkflow(options: UseScanMoveWorkflowOptions) { plan.events.forEach(dispatch) return Promise.resolve() } - return runBusy(() => resolveAgainst(collections, false)) - }, [state, collections, variant, runBusy, resolveAgainst]) + return runExclusive(() => resolveAgainst(collections, false)) + }, [state, collections, variant, runExclusive, resolveAgainst]) /** Resolve against a freshly refetched collection list (post-create flows). */ const resolveWithFreshCollections = useCallback( () => - runBusy(async () => { + runExclusive(async () => { const fresh = refreshCollections ? await refreshCollections() : collections await resolveAgainst(fresh, true) }), - [runBusy, refreshCollections, collections, resolveAgainst], + [runExclusive, refreshCollections, collections, resolveAgainst], ) /** Create all pending destinations, then resolve with the fresh collection list. */ @@ -198,7 +223,7 @@ export function useScanMoveWorkflow(options: UseScanMoveWorkflowOptions) { }) return Promise.resolve() } - return runBusy(async () => { + return runExclusive(async () => { const { allSuccess } = await createScanMoveDestinations( state.pendingDestinations, gateway, @@ -208,16 +233,16 @@ export function useScanMoveWorkflow(options: UseScanMoveWorkflowOptions) { const fresh = refreshCollections ? await refreshCollections() : collections await resolveAgainst(fresh, true) }) - }, [state.pendingDestinations, runBusy, gateway, refreshCollections, collections, resolveAgainst]) + }, [state.pendingDestinations, runExclusive, gateway, refreshCollections, collections, resolveAgainst]) const executeMoves = useCallback( () => - runBusy(async () => { + runExclusive(async () => { dispatch(await executeScanMove(variant, state, gateway)) // Best-effort moves can succeed partially; refresh plate, box, and location views. void invalidateAfterBulkWrite(queryClient) }), - [runBusy, variant, state, gateway, queryClient], + [runExclusive, variant, state, gateway, queryClient], ) const reset = useCallback(() => { diff --git a/packages/web/src/lib/__tests__/scanner-plate-csv.test.ts b/packages/web/src/lib/__tests__/scanner-plate-csv.test.ts index e558f79f..e49b31f2 100644 --- a/packages/web/src/lib/__tests__/scanner-plate-csv.test.ts +++ b/packages/web/src/lib/__tests__/scanner-plate-csv.test.ts @@ -85,7 +85,52 @@ describe('validateScannerPlateCsv', () => { const rows = parseScannerPlateCsv(text, singleConfig) const result = validateScannerPlateCsv(rows, singleConfig) expect(result.valid).toBe(false) - expect(result.errors[0].error).toMatch(/Found 95 valid position/) + expect(result.errors).toContainEqual({ row: 96, error: 'Position A01 appears more than once (also row 1)' }) + expect(result.errors.some((e) => /Found 95 valid position/.test(e.error))).toBe(true) + }) + + // Traxcer exports (examples/move-micronix-test): "Tube ID,Position", unpadded wells. + const traxcerConfig: ScannerConfiguration = { + id: 'cfg-traxcer', + name: 'Traxcer', + barcodeColumn: 'Tube ID', + positionType: 'single', + positionColumn: 'Position', + skipRows: 0, + isDefault: true, + } + + /** Full plate in Traxcer form with unpadded positions (A1 … H12). */ + function traxcerPlate(barcodes: Record, positionOverride: Record = {}): string { + const lines = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'].flatMap((row) => + Array.from({ length: 12 }, (_, i) => { + const pos = `${row}${i + 1}` + return `${barcodes[pos] ?? ''},${positionOverride[pos] ?? pos}` + }), + ) + return 'Tube ID,Position\n' + lines.join('\n') + } + + it('accepts unpadded scanner positions like A1', () => { + const rows = parseScannerPlateCsv(traxcerPlate({ A1: '1043659134', A2: '1106665111' }), traxcerConfig) + expect(validateScannerPlateCsv(rows, traxcerConfig)).toEqual({ valid: true, errors: [] }) + }) + + it('catches A1 and A01 in one scan as the same well', () => { + // Row 2 (A2) is written as a01, so A01 appears twice and A02 is missing. + const rows = parseScannerPlateCsv(traxcerPlate({}, { A2: 'a01' }), traxcerConfig) + const result = validateScannerPlateCsv(rows, traxcerConfig) + expect(result.valid).toBe(false) + expect(result.errors).toContainEqual({ row: 2, error: 'Position A01 appears more than once (also row 1)' }) + }) + + it('rejects the same barcode in two wells', () => { + const rows = parseScannerPlateCsv(traxcerPlate({ A1: '1043659134', C7: '1043659134' }), traxcerConfig) + const result = validateScannerPlateCsv(rows, traxcerConfig) + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { row: 31, error: 'Barcode 1043659134 appears more than once (also row 1)' }, + ]) }) it('rejects invalid well positions like I01', () => { diff --git a/packages/web/src/lib/scan-move/__tests__/execute.test.ts b/packages/web/src/lib/scan-move/__tests__/execute.test.ts index 221fad3a..244c19ad 100644 --- a/packages/web/src/lib/scan-move/__tests__/execute.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/execute.test.ts @@ -27,8 +27,8 @@ describe('buildMovePlan', () => { const plan = buildMovePlan(files, micronixScanMoveVariant) if (!plan.ok) throw new Error('expected ok plan') expect(plan.moves).toEqual([ - { identifier: { type: 'barcode', barcode: 'BC1' }, targetPosition: 'A01', fileIndex: 0 }, - { identifier: { type: 'barcode', barcode: 'BC2' }, targetPosition: 'B01', fileIndex: 1 }, + { identifier: { type: 'barcode', barcode: 'BC1' }, targetPosition: 'A01', fileIndex: 0, csvRow: 1 }, + { identifier: { type: 'barcode', barcode: 'BC2' }, targetPosition: 'B01', fileIndex: 1, csvRow: 1 }, ]) expect(plan.mappings).toEqual([{ fromCollectionName: 'SRC-1', toCollectionName: 'DEST-1' }]) }) @@ -57,43 +57,86 @@ describe('buildMovePlan', () => { }) describe('buildPerFileResults', () => { - it('attributes moved counts and row-ranged errors per file', () => { - const files = [ - makeFile({ - filename: 'one.csv', - selectedDestinationName: 'DEST-1', - csvRows: [ - { container_barcode: 'BC1', target_position: 'A01' }, - { container_barcode: 'BC2', target_position: 'A02' }, - ], - }), - makeFile({ - filename: 'two.csv', - selectedDestinationName: 'DEST-2', - csvRows: [{ container_barcode: 'BC3', target_position: 'B01' }], - }), - ] - const moves = [ - { identifier: { type: 'barcode' as const, barcode: 'BC1' }, targetPosition: 'A01', fileIndex: 0 }, - { identifier: { type: 'barcode' as const, barcode: 'BC2' }, targetPosition: 'A02', fileIndex: 0 }, - { identifier: { type: 'barcode' as const, barcode: 'BC3' }, targetPosition: 'B01', fileIndex: 1 }, - ] - const results = buildPerFileResults(files, moves, { + // Scanner files list empty wells; the move request skips them, so the + // endpoint's row numbers don't line up with CSV rows. + const files = [ + makeFile({ + filename: 'one.csv', + selectedDestinationName: 'DEST-1', + csvRows: [ + { container_barcode: 'BC1', target_position: 'A01' }, + { container_barcode: '', target_position: 'A02' }, + { container_barcode: 'BC2', target_position: 'A03' }, + ], + }), + makeFile({ + filename: 'two.csv', + selectedDestinationName: 'DEST-2', + csvRows: [ + { container_barcode: '', target_position: 'A01' }, + { container_barcode: 'BC3', target_position: 'B01' }, + ], + }), + ] + const plan = buildMovePlan(files, micronixScanMoveVariant) + if (!plan.ok) throw new Error('unexpected plan failure') + + it('records each move with its file and CSV row', () => { + expect(plan.moves.map((m) => [m.fileIndex, m.csvRow])).toEqual([ + [0, 1], + [0, 3], + [1, 2], + ]) + }) + + it('maps request rows back to the file and CSV row they came from', () => { + const { fileResults, unattributed } = buildPerFileResults(files, plan.moves, { success: false, - errors: [{ row: 2, error: 'occupied' }], + moved: 0, + errors: [ + { row: 2, error: 'occupied' }, + { row: 3, error: 'not found' }, + { row: 0, error: 'Invalid collection mappings' }, + ], }) - expect(results).toEqual([ + expect(fileResults).toEqual([ { filename: 'one.csv', destinationName: 'DEST-1', moved: 0, - errors: [{ row: 2, error: 'occupied' }], + errors: [{ row: 3, error: 'occupied', filename: 'one.csv' }], + }, + { + filename: 'two.csv', + destinationName: 'DEST-2', + moved: 0, + errors: [{ row: 2, error: 'not found', filename: 'two.csv' }], }, - { filename: 'two.csv', destinationName: 'DEST-2', moved: 0, errors: [] }, ]) + expect(unattributed).toEqual([{ row: 0, error: 'Invalid collection mappings' }]) + }) + + it('splits a best-effort partial result per file', () => { + const { fileResults } = buildPerFileResults(files, plan.moves, { + success: false, + moved: 2, + errors: [{ row: 3, error: 'not found' }], + }) + expect(fileResults.map((r) => r.moved)).toEqual([2, 0]) + }) + + it('leaves per-file counts unknown when a partial result does not add up', () => { + const { fileResults } = buildPerFileResults(files, plan.moves, { + success: false, + moved: 1, + errors: [{ row: 3, error: 'failed mid-way' }], + }) + expect(fileResults.map((r) => r.moved)).toEqual([undefined, undefined]) + }) - const successResults = buildPerFileResults(files, moves, { success: true }) - expect(successResults.map((r) => r.moved)).toEqual([2, 1]) + it('counts every move on success', () => { + const { fileResults } = buildPerFileResults(files, plan.moves, { success: true, moved: 3 }) + expect(fileResults.map((r) => r.moved)).toEqual([2, 1]) }) }) @@ -129,8 +172,9 @@ describe('executeScanMove', () => { if (event.type !== 'MOVE_COMPLETED') throw new Error('unexpected event') expect(event.result).toMatchObject({ success: true, moved: 1 }) expect(event.result.fileResults).toEqual([ - { filename: 'one.csv', destinationName: 'DEST-1', moved: 1, errors: undefined }, + { filename: 'one.csv', destinationName: 'DEST-1', moved: 1, errors: [] }, ]) + expect(event.result.errors).toBeUndefined() }) it('returns a failed result without calling the API when the plan conflicts', async () => { @@ -166,7 +210,7 @@ describe('executeScanMove', () => { Promise.reject( Object.assign(new Error('Request failed'), { response: { - data: { error: 'move failed', moved: 2, errors: [{ row: 3, error: 'occupied' }] }, + data: { error: 'move failed', moved: 0, errors: [{ row: 1, error: 'occupied' }, { row: 9, error: 'odd' }] }, }, }), ), @@ -175,9 +219,31 @@ describe('executeScanMove', () => { if (event.type !== 'MOVE_COMPLETED') throw new Error('unexpected event') expect(event.result).toEqual({ success: false, - moved: 2, - errors: [{ row: 3, error: 'occupied' }], + moved: 0, + // A row past the request can't be placed in a file. + errors: [{ row: 0, error: 'odd' }], + fileResults: [ + { + filename: 'one.csv', + destinationName: 'DEST-1', + moved: 0, + errors: [{ row: 1, error: 'occupied', filename: 'one.csv' }], + }, + ], + }) + }) + + it('reports per-file errors when the move endpoint returns success: false', async () => { + const gateway = stubGateway({ + moveContainers: () => + Promise.resolve({ success: false, moved: 0, errors: [{ row: 1, error: 'occupied' }] }), }) + const event = await executeScanMove(micronixScanMoveVariant, makeState({ files }), gateway) + if (event.type !== 'MOVE_COMPLETED') throw new Error('unexpected event') + expect(event.result.errors).toBeUndefined() + expect(event.result.fileResults?.[0].errors).toEqual([ + { row: 1, error: 'occupied', filename: 'one.csv' }, + ]) }) it('falls back to the error message when no body is present', async () => { diff --git a/packages/web/src/lib/scan-move/__tests__/ingest.test.ts b/packages/web/src/lib/scan-move/__tests__/ingest.test.ts index 3b938cae..d3a1d720 100644 --- a/packages/web/src/lib/scan-move/__tests__/ingest.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/ingest.test.ts @@ -96,6 +96,20 @@ describe('cryovial ingest', () => { ]) }) + it('rejects the same target position or source tube twice in one file', () => { + const text = [ + 'source_collection_name,source_position,target_position', + 'BOX-BETA,A01,B01', + 'BOX-BETA,A02,b1', + 'BOX-BETA,a1,C01', + ].join('\n') + const file = ingestScanCsvText(cryovialScanMoveVariant, fileSource('box-alpha.csv'), text, { collections: boxes }) + expect(file.validationErrors).toEqual([ + { row: 2, error: 'Target position B01 appears more than once (also row 1)' }, + { row: 3, error: 'Source tube BOX-BETA A01 appears more than once (also row 1)' }, + ]) + }) + it('requires a manual choice when multiple boxes match the filename case-insensitively', () => { const text = 'source_collection_name,source_position,target_position\nBOX-BETA,A01,B01' const file = ingestScanCsvText(cryovialScanMoveVariant, fileSource('box-alpha.csv'), text, { diff --git a/packages/web/src/lib/scan-move/__tests__/reducer.test.ts b/packages/web/src/lib/scan-move/__tests__/reducer.test.ts index 90bc3521..bb6bdeb8 100644 --- a/packages/web/src/lib/scan-move/__tests__/reducer.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/reducer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { initialScanMoveState, planNextFromUpload, scanMoveReducer } from '../reducer' +import { blockingErrors, initialScanMoveState, planNextFromUpload, scanMoveReducer } from '../reducer' import { cryovialScanMoveVariant, micronixScanMoveVariant } from '../variants' import { makeFile, makeState } from './helpers' @@ -86,20 +86,57 @@ describe('scanMoveReducer', () => { advanced: true, }) expect(advanced.step).toBe('resolve') - // No new errors: the stale relocation error from the prior attempt stays untouched - // only when the outcome carries errors; a clean outcome leaves prior errors as-is. expect(advanced.files[0].resolvedContainers).toHaveLength(1) + // A clean resolve clears the stale error from the previous attempt. + expect(advanced.files[0].validationErrors).toEqual([]) }) - it('RESOLVE_FAILED appends the failure to every file without advancing', () => { + it('RESOLVE_FAILED tags the failure as a lookup error on every file without advancing', () => { const state = makeState({ files: [makeFile(), makeFile({ filename: 'b.csv' })] }) const next = scanMoveReducer(state, { type: 'RESOLVE_FAILED', message: 'boom' }) expect(next.step).toBe('upload') for (const file of next.files) { - expect(file.validationErrors).toEqual([{ row: 0, error: 'boom' }]) + expect(file.validationErrors).toEqual([ + { row: 0, error: 'Lookup failed: boom. Select Next to try again.', kind: 'lookup' }, + ]) } }) + it('a failed lookup leaves Next able to retry, without stacking errors', () => { + const file = makeFile({ + selectedDestinationName: 'PLATE-1', + csvRows: [{ container_barcode: 'BC1', target_position: 'A01' }], + }) + const plates = [{ id: 1, name: 'PLATE-1' }] + let state = makeState({ files: [file] }) + state = scanMoveReducer(state, { type: 'RESOLVE_FAILED', message: 'Request failed with status code 500' }) + expect(blockingErrors(state.files[0])).toEqual([]) + expect(planNextFromUpload(state, plates, micronixScanMoveVariant).kind).toBe('resolve') + + state = scanMoveReducer(state, { type: 'RESOLVE_FAILED', message: 'again' }) + expect(state.files[0].validationErrors).toHaveLength(1) + expect(planNextFromUpload(state, plates, micronixScanMoveVariant).kind).toBe('resolve') + + state = scanMoveReducer(state, { + type: 'RESOLVE_COMPLETED', + outcomes: [{ fileIndex: 0, resolvedContainers: [], unresolvedContainers: [], errors: [] }], + advanced: true, + }) + expect(state.step).toBe('resolve') + expect(state.files[0].validationErrors).toEqual([]) + }) + + it('upload format errors still block Next after a failed lookup', () => { + const file = makeFile({ + selectedDestinationName: 'PLATE-1', + validationErrors: [{ row: 3, error: 'Position A01 appears more than once (also row 1)' }], + }) + const state = scanMoveReducer(makeState({ files: [file] }), { type: 'RESOLVE_FAILED', message: 'boom' }) + expect(planNextFromUpload(state, [{ id: 1, name: 'PLATE-1' }], micronixScanMoveVariant).kind).toBe( + 'has_errors', + ) + }) + it('MOVE_COMPLETED stores the result and lands on execute', () => { const state = makeState({ step: 'resolve', files: [makeFile()] }) const next = scanMoveReducer(state, { diff --git a/packages/web/src/lib/scan-move/__tests__/resolve.test.ts b/packages/web/src/lib/scan-move/__tests__/resolve.test.ts index 640f1c44..dd00c956 100644 --- a/packages/web/src/lib/scan-move/__tests__/resolve.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/resolve.test.ts @@ -91,7 +91,7 @@ describe('groupResolveResults', () => { micronixScanMoveVariant, ) expect(outcomes[0].errors).toEqual([ - { row: 0, error: 'Some containers are not from micronix plates' }, + { row: 0, error: 'Some containers are not from micronix plates', kind: 'resolution' }, ]) expect(outcomes[1].errors).toEqual([]) }) @@ -114,11 +114,11 @@ describe('validateRelocations', () => { new Map([ ['PLATE-A', { A02: { type: 'micronix_tube', barcode: 'ORPHAN' } }], ]), - 'plate', + micronixScanMoveVariant, ) expect(errors.get(0)).toEqual([ { - row: 0, + row: 2, error: 'Position A02 on plate "PLATE-A" is empty in your upload but tube ORPHAN is currently there and is not relocated in this move.', kind: 'relocation', @@ -141,13 +141,68 @@ describe('validateRelocations', () => { new Map([ ['PLATE-A', { A02: { type: 'micronix_tube', barcode: 'ORPHAN' } }], ]), - 'plate', + micronixScanMoveVariant, + ) + expect(errors.size).toBe(0) + }) + + it('matches scanner positions like A1 and a01 to the stored A01 well', () => { + const files = [ + makeFile({ + selectedDestinationName: 'PLATE-A', + csvRows: [ + { container_barcode: 'BC1', target_position: 'b1' }, + { container_barcode: '', target_position: 'A1' }, + { container_barcode: '', target_position: 'a03' }, + ], + }), + ] + const errors = validateRelocations( + files, + new Map([ + [ + 'PLATE-A', + { + A01: { type: 'micronix_tube', barcode: 'ORPHAN-1' }, + // A legacy unpadded key still matches. + A3: { type: 'micronix_tube', barcode: 'ORPHAN-3' }, + }, + ], + ]), + micronixScanMoveVariant, + ) + expect(errors.get(0)?.map((e) => [e.row, e.error])).toEqual([ + [ + 2, + 'Position A01 on plate "PLATE-A" is empty in your upload but tube ORPHAN-1 is currently there and is not relocated in this move.', + ], + [ + 3, + 'Position A03 on plate "PLATE-A" is empty in your upload but tube ORPHAN-3 is currently there and is not relocated in this move.', + ], + ]) + }) + + it('treats a tube scanned at B1 as filling the stored B01 well', () => { + const files = [ + makeFile({ + selectedDestinationName: 'PLATE-A', + csvRows: [ + { container_barcode: 'BC1', target_position: 'B1' }, + { container_barcode: '', target_position: 'B01' }, + ], + }), + ] + const errors = validateRelocations( + files, + new Map([['PLATE-A', { B01: { type: 'micronix_tube', barcode: 'OTHER' } }]]), + micronixScanMoveVariant, ) expect(errors.size).toBe(0) }) it('reports a missing destination instead of validating wells', () => { - const errors = validateRelocations(baseFiles, new Map(), 'plate') + const errors = validateRelocations(baseFiles, new Map(), micronixScanMoveVariant) expect(errors.get(0)).toEqual([ { row: 0, diff --git a/packages/web/src/lib/scan-move/execute.ts b/packages/web/src/lib/scan-move/execute.ts index 7a0af7c8..44353d5a 100644 --- a/packages/web/src/lib/scan-move/execute.ts +++ b/packages/web/src/lib/scan-move/execute.ts @@ -13,6 +13,8 @@ export interface ScanMovePlannedMove { identifier: ScanMoveIdentifier targetPosition: string fileIndex: number + /** 1-based CSV data row in that file, as upload errors number rows. */ + csvRow: number } export type ScanMovePlan = @@ -30,10 +32,10 @@ export type ScanMovePlan = export function buildMovePlan(files: ScanMoveFile[], variant: ScanMoveVariant): ScanMovePlan { const moves: ScanMovePlannedMove[] = [] files.forEach((file, fileIndex) => { - file.csvRows.forEach((row) => { + file.csvRows.forEach((row, rowIndex) => { const identifier = variant.identifierFromRow(row) if (identifier === null) return - moves.push({ identifier, targetPosition: row.target_position.trim(), fileIndex }) + moves.push({ identifier, targetPosition: row.target_position.trim(), fileIndex, csvRow: rowIndex + 1 }) }) }) @@ -68,24 +70,69 @@ export function buildMovePlan(files: ScanMoveFile[], variant: ScanMoveVariant): } } +interface MoveResponseError { + row: number + error: string +} + +/** + * The move endpoint numbers error rows by position in the request's moves + * list, which spans every file and skips empty wells. Map each error back to + * the file and CSV row it came from; errors without a usable row stay + * unattributed. + */ +function attributeErrors( + files: ScanMoveFile[], + moves: ScanMovePlannedMove[], + errors: MoveResponseError[], +): { byFile: ScanMoveValidationError[][]; unattributed: ScanMoveValidationError[] } { + const byFile: ScanMoveValidationError[][] = files.map(() => []) + const unattributed: ScanMoveValidationError[] = [] + for (const error of errors) { + const move = Number.isInteger(error.row) && error.row > 0 ? moves[error.row - 1] : undefined + if (move === undefined) { + unattributed.push({ row: 0, error: error.error }) + continue + } + byFile[move.fileIndex].push({ + row: move.csvRow, + error: error.error, + filename: files[move.fileIndex].filename, + }) + } + return { byFile, unattributed } +} + /** - * Map the move response back onto files. Error-to-file attribution is by - * row range, as the move endpoint does not echo file boundaries. + * Map a move response, successful or not, back onto files: per-file moved + * counts and errors, plus the errors no row accounts for. */ export function buildPerFileResults( files: ScanMoveFile[], moves: ScanMovePlannedMove[], - response: { success: boolean; errors?: Array<{ row: number; error: string }> }, -): ScanMovePerFileResult[] { - return files.map((file, fileIndex) => { - const fileMoves = moves.filter((m) => m.fileIndex === fileIndex) + response: { success: boolean; moved?: number; errors?: MoveResponseError[] }, +): { fileResults: ScanMovePerFileResult[]; unattributed: ScanMoveValidationError[] } { + const { byFile, unattributed } = attributeErrors(files, moves, response.errors ?? []) + const erroredRows = new Set( + (response.errors ?? []).map((e) => e.row).filter((row) => row > 0 && row <= moves.length), + ) + // A partial result splits per file only when every non-errored move is + // accounted for (best effort); otherwise the per-file count is unknown. + const partialSplits = !response.success && (response.moved ?? 0) === moves.length - erroredRows.size + const fileResults = files.map((file, fileIndex): ScanMovePerFileResult => { + const fileMoveCount = moves.filter((m) => m.fileIndex === fileIndex).length + let moved: number | undefined + if (response.success) moved = fileMoveCount + else if ((response.moved ?? 0) === 0) moved = 0 + else if (partialSplits) moved = fileMoveCount - byFile[fileIndex].length return { filename: file.filename, destinationName: file.selectedDestinationName ?? '', - moved: response.success ? fileMoves.length : 0, - errors: response.errors?.filter((e) => e.row > 0 && e.row <= file.csvRows.length), + moved, + errors: byFile[fileIndex], } }) + return { fileResults, unattributed } } /** Execute effect: build the plan, run the move, normalize errors. */ @@ -109,32 +156,44 @@ export async function executeScanMove( mappings: plan.mappings, moves: plan.moves.map(({ identifier, targetPosition }) => ({ identifier, targetPosition })), }) + const { fileResults, unattributed } = buildPerFileResults(state.files, plan.moves, response) return { type: 'MOVE_COMPLETED', result: { success: response.success, moved: response.moved || 0, - errors: response.success ? undefined : response.errors, - fileResults: buildPerFileResults(state.files, plan.moves, response), + errors: response.success || unattributed.length === 0 ? undefined : unattributed, + fileResults, }, } } catch (err) { // Standardized backend error body: { error, moved, errors } const data = err && typeof err === 'object' && 'response' in err - ? ((err as { response?: { data?: { error?: string; moved?: number; errors?: ScanMoveValidationError[] } } }) + ? ((err as { response?: { data?: { error?: string; moved?: number; errors?: MoveResponseError[] } } }) .response?.data ?? {}) : {} - const errors: ScanMoveValidationError[] = data.errors ?? [] - if (errors.length === 0) { - errors.push({ + const moved = data.moved ?? 0 + const { fileResults, unattributed } = buildPerFileResults(state.files, plan.moves, { + success: false, + moved, + errors: data.errors ?? [], + }) + const attributedCount = fileResults.reduce((n, r) => n + (r.errors?.length ?? 0), 0) + if (unattributed.length === 0 && attributedCount === 0) { + unattributed.push({ row: 0, error: data.error || (err instanceof Error ? err.message : '') || 'Failed to move containers', }) } return { type: 'MOVE_COMPLETED', - result: { success: false, moved: data.moved ?? 0, errors }, + result: { + success: false, + moved, + errors: unattributed.length > 0 ? unattributed : undefined, + fileResults, + }, } } } diff --git a/packages/web/src/lib/scan-move/index.ts b/packages/web/src/lib/scan-move/index.ts index f06c2017..58e1390a 100644 --- a/packages/web/src/lib/scan-move/index.ts +++ b/packages/web/src/lib/scan-move/index.ts @@ -1,5 +1,5 @@ export * from './types' -export { initialScanMoveState, scanMoveReducer, planNextFromUpload } from './reducer' +export { initialScanMoveState, scanMoveReducer, planNextFromUpload, blockingErrors } from './reducer' export type { ScanMoveUploadPlan } from './reducer' export { splitCsvLine, diff --git a/packages/web/src/lib/scan-move/reducer.ts b/packages/web/src/lib/scan-move/reducer.ts index 34ed61ef..18a11eb5 100644 --- a/packages/web/src/lib/scan-move/reducer.ts +++ b/packages/web/src/lib/scan-move/reducer.ts @@ -5,6 +5,7 @@ import type { ScanMoveFile, ScanMoveFileErrors, ScanMoveState, + ScanMoveValidationError, ScanMoveVariant, } from './types' @@ -19,6 +20,25 @@ export function initialScanMoveState(): ScanMoveState { } } +/** Errors a resolve produces; each resolve (or failed attempt) replaces them. */ +const RESOLVE_ERROR_KINDS: ReadonlySet = new Set([ + 'relocation', + 'resolution', + 'lookup', +]) + +function withoutResolveErrors(errors: ScanMoveValidationError[]): ScanMoveValidationError[] { + return errors.filter((e) => !RESOLVE_ERROR_KINDS.has(e.kind)) +} + +/** + * Errors that stop the upload step's Next. A failed lookup doesn't: Next + * retries the lookup with the files already uploaded. + */ +export function blockingErrors(file: ScanMoveFile): ScanMoveValidationError[] { + return file.validationErrors.filter((e) => e.kind !== 'lookup') +} + function clearResolution(file: ScanMoveFile, name: string | null): ScanMoveFile { return { ...file, @@ -104,19 +124,12 @@ export function scanMoveReducer(state: ScanMoveState, event: ScanMoveEvent): Sca const files = state.files.map((f, i) => { const outcome = byIndex.get(i) if (!outcome) return f - const base = { + return { ...f, resolvedContainers: outcome.resolvedContainers, unresolvedContainers: outcome.unresolvedContainers, isResolved: true, - } - if (outcome.errors.length === 0) return base - return { - ...base, - validationErrors: [ - ...f.validationErrors.filter((e) => e.kind !== 'relocation'), - ...outcome.errors, - ], + validationErrors: [...withoutResolveErrors(f.validationErrors), ...outcome.errors], } }) return guarded({ ...state, files, step: event.advanced ? 'resolve' : state.step }) @@ -127,7 +140,10 @@ export function scanMoveReducer(state: ScanMoveState, event: ScanMoveEvent): Sca ...state, files: state.files.map((f) => ({ ...f, - validationErrors: [...f.validationErrors, { row: 0, error: event.message }], + validationErrors: [ + ...withoutResolveErrors(f.validationErrors), + { row: 0, error: `Lookup failed: ${event.message}. Select Next to try again.`, kind: 'lookup' }, + ], })), } @@ -176,7 +192,7 @@ export function planNextFromUpload( } } - if (state.files.some((f) => f.validationErrors.length > 0)) { + if (state.files.some((f) => blockingErrors(f).length > 0)) { return { kind: 'has_errors', events: [{ type: 'STEP_SET', step: 'upload' }] } } diff --git a/packages/web/src/lib/scan-move/resolve.ts b/packages/web/src/lib/scan-move/resolve.ts index 2aacc504..666ef26b 100644 --- a/packages/web/src/lib/scan-move/resolve.ts +++ b/packages/web/src/lib/scan-move/resolve.ts @@ -1,3 +1,4 @@ +import { parseGridPosition, type GridCollectionType } from '@sampledb/contract' import type { ScanMoveCollectionRef, ScanMoveEvent, @@ -86,22 +87,35 @@ export function groupResolveResults( } for (const fileIndex of wrongTypeFileIndexes) { - outcomes[fileIndex].errors.push({ row: 0, error: variant.wrongCollectionTypeError }) + outcomes[fileIndex].errors.push({ row: 0, error: variant.wrongCollectionTypeError, kind: 'resolution' }) } return outcomes } +type DestinationWells = Record + +/** + * Stored-form (`A01`) key for a scanned or stored position, or null when it + * isn't a Grid position. Scanners write `A1` or `a01`; wells are keyed `A01`. + */ +function gridKey(position: string, collectionType: GridCollectionType): string | null { + const parsed = parseGridPosition(position, collectionType) + return parsed.ok ? parsed.position : null +} + /** * Relocation validation (micronix capability): a position that is empty in * the upload but currently holds a tube is only valid when that tube is - * relocated elsewhere in the same scan move. + * relocated elsewhere in the same scan move. Positions on both sides are + * compared in stored form by the Grid position rule. */ export function validateRelocations( files: ScanMoveFile[], - wellsByDestinationName: Map>, - destinationNoun: string, + wellsByDestinationName: Map, + variant: Pick, ): Map { + const { destinationNoun, collectionType } = variant const errorsByFile = new Map() const pushError = (fileIndex: number, error: ScanMoveValidationError) => { if (!errorsByFile.has(fileIndex)) errorsByFile.set(fileIndex, []) @@ -126,32 +140,36 @@ export function validateRelocations( continue } + const wellsByPosition = new Map() + for (const [position, well] of Object.entries(wells)) { + wellsByPosition.set(gridKey(position, collectionType) ?? position, well) + } + const positionToBarcode = new Map() - const positionToEmptyFileIndex = new Map() + const positionToEmpty = new Map() files.forEach((file, fileIndex) => { if (file.selectedDestinationName !== destinationName) return - for (const row of file.csvRows) { - const position = row.target_position.trim() + file.csvRows.forEach((row, rowIndex) => { + const raw = row.target_position.trim() const barcode = row.container_barcode.trim() - if (position === '') continue + if (raw === '') return + // A position off the grid is reported at upload; compare it verbatim here. + const position = gridKey(raw, collectionType) ?? raw if (barcode !== '') { positionToBarcode.set(position, barcode) - } else if (!positionToEmptyFileIndex.has(position)) { - positionToEmptyFileIndex.set(position, fileIndex) + } else if (!positionToEmpty.has(position)) { + positionToEmpty.set(position, { fileIndex, row: rowIndex + 1 }) } - } + }) }) const barcodesRelocatedInMove = new Set(positionToBarcode.values()) - const emptyPositions = [...positionToEmptyFileIndex.keys()].filter( - (position) => !positionToBarcode.has(position), - ) - - for (const position of emptyPositions) { - const well = wells[position] as { type: string; barcode?: string | null } | undefined + for (const [position, origin] of positionToEmpty) { + if (positionToBarcode.has(position)) continue + const well = wellsByPosition.get(position) if (well?.type === 'micronix_tube' && well.barcode && !barcodesRelocatedInMove.has(well.barcode)) { - pushError(positionToEmptyFileIndex.get(position) ?? 0, { - row: 0, + pushError(origin.fileIndex, { + row: origin.row, error: `Position ${position} on ${destinationNoun} "${destinationName}" is empty in your upload but tube ${well.barcode} is currently there and is not relocated in this move.`, kind: 'relocation', }) @@ -183,20 +201,13 @@ export async function resolveScanMove( files.map((f) => f.selectedDestinationName).filter((n): n is string => Boolean(n)), ), ] - const wellsByDestinationName = new Map< - string, - Record - >() + const wellsByDestinationName = new Map() for (const name of destinationNames) { const collection = collections.find((c) => c.name === name) if (collection === undefined) continue wellsByDestinationName.set(name, await gateway.getDestinationWells(collection.id)) } - const relocationErrors = validateRelocations( - files, - wellsByDestinationName, - variant.destinationNoun, - ) + const relocationErrors = validateRelocations(files, wellsByDestinationName, variant) for (const [fileIndex, errors] of relocationErrors) { outcomes[fileIndex].errors.push(...errors) } diff --git a/packages/web/src/lib/scan-move/types.ts b/packages/web/src/lib/scan-move/types.ts index e364744b..da3ff723 100644 --- a/packages/web/src/lib/scan-move/types.ts +++ b/packages/web/src/lib/scan-move/types.ts @@ -22,15 +22,20 @@ export type ScanMoveContainerInfo = ContainerMoveContainerInfo export type PendingDestination = PendingDestinationPlate export interface ScanMoveValidationError { + /** 1-based CSV data row within `filename` (or the file it is listed under); 0 = not row-specific. */ row: number error: string + /** CSV file the row belongs to, when the error is reported outside that file's own list. */ + filename?: string /** * Structural tag so consumers can act on error provenance instead of - * matching message text: the reducer drops stale 'destination' errors on + * matching message text: the reducer drops stale kinded errors on * re-selection, and plate scan validation surfaces only 'inference' errors * (format rules like the full-plate check don't apply there). + * - 'relocation' / 'resolution': found by a resolve; replaced by the next one. + * - 'lookup': the resolve request itself failed; Next retries it. */ - kind?: 'relocation' | 'destination' | 'inference' + kind?: 'relocation' | 'resolution' | 'lookup' | 'destination' | 'inference' } export type ScanMoveIdentifier = @@ -72,7 +77,8 @@ export interface ScanMoveFile { export interface ScanMovePerFileResult { filename: string destinationName: string - moved: number + /** Tubes moved from this file; undefined when a partial result can't be split per file. */ + moved?: number errors?: ScanMoveValidationError[] } diff --git a/packages/web/src/lib/scan-move/variants.ts b/packages/web/src/lib/scan-move/variants.ts index 2da9dfa6..695a69be 100644 --- a/packages/web/src/lib/scan-move/variants.ts +++ b/packages/web/src/lib/scan-move/variants.ts @@ -1,6 +1,6 @@ import { parseGridPosition } from '@sampledb/contract' import { inferDestinationPlateForScan } from '../plate-destination-inference' -import { parseScannerPlateCsv, validateScannerPlateCsv } from '../scanner-plate-csv' +import { findDuplicateRows, parseScannerPlateCsv, validateScannerPlateCsv } from '../scanner-plate-csv' import { parseBuiltinMoveCsv } from './ingest' import type { ScanMoveCsvRow, ScanMoveIdentifier, ScanMoveInference, ScanMoveVariant } from './types' @@ -92,12 +92,31 @@ export const cryovialScanMoveVariant: ScanMoveVariant = { parseAndValidate(text) { const { csvRows, errors } = parseBuiltinMoveCsv(text, CRYOVIAL_MOVE_CSV_SPEC) // Grid position rule on the target only: a legacy source position must still resolve. - csvRows.forEach((row, i) => { + const targets = csvRows.map((row, i) => { const target = csvCell(row, 'target_position') - if (target === '') return + if (target === '') return null const parsed = parseGridPosition(target, 'cryovial_box') - if (!parsed.ok) errors.push({ row: i + 1, error: parsed.error }) + if (parsed.ok) return parsed.position + errors.push({ row: i + 1, error: parsed.error }) + return null }) + errors.push( + ...findDuplicateRows( + targets, + (position, firstRow) => `Target position ${position} appears more than once (also row ${firstRow})`, + ), + // The same source tube listed twice would be moved twice. + ...findDuplicateRows( + csvRows.map((row) => { + const box = csvCell(row, 'source_collection_name') + const position = csvCell(row, 'source_position') + if (box === '' || position === '') return null + const parsed = parseGridPosition(position, 'cryovial_box') + return `${box} ${parsed.ok ? parsed.position : position}` + }), + (source, firstRow) => `Source tube ${source} appears more than once (also row ${firstRow})`, + ), + ) return { csvRows, errors } }, diff --git a/packages/web/src/lib/scanner-plate-csv.ts b/packages/web/src/lib/scanner-plate-csv.ts index 206ee904..59cf2ff6 100644 --- a/packages/web/src/lib/scanner-plate-csv.ts +++ b/packages/web/src/lib/scanner-plate-csv.ts @@ -49,6 +49,25 @@ export interface ScannerPlateCsvValidationError { error: string } +/** + * One error per repeat of a value: `keys[i]` is row i+1's key (null = row not + * checked). The error names the row of the first occurrence. + */ +export function findDuplicateRows( + keys: Array, + describe: (key: string, firstRow: number) => string, +): ScannerPlateCsvValidationError[] { + const firstRowByKey = new Map() + const errors: ScannerPlateCsvValidationError[] = [] + keys.forEach((key, i) => { + if (key === null || key === '') return + const firstRow = firstRowByKey.get(key) + if (firstRow === undefined) firstRowByKey.set(key, i + 1) + else errors.push({ row: i + 1, error: describe(key, firstRow) }) + }) + return errors +} + export function validateScannerPlateCsv( rows: ScannerPlateCsvRow[], config: ScannerConfiguration @@ -73,6 +92,18 @@ export function validateScannerPlateCsv( } } + // A1 and A01 are the same well; compare in stored form. + errors.push( + ...findDuplicateRows( + rows.map((row) => normalizeWellPosition(row.target_position) ?? (row.target_position.trim() || null)), + (position, firstRow) => `Position ${position} appears more than once (also row ${firstRow})`, + ), + ...findDuplicateRows( + rows.map((row) => row.container_barcode.trim() || null), + (barcode, firstRow) => `Barcode ${barcode} appears more than once (also row ${firstRow})`, + ), + ) + const positionSet = new Set() for (const row of rows) { const pos = row.target_position.trim() diff --git a/packages/web/src/pages/__tests__/ContainerMoveCryovial.test.tsx b/packages/web/src/pages/__tests__/ContainerMoveCryovial.test.tsx index b01abc2a..d52b0a95 100644 --- a/packages/web/src/pages/__tests__/ContainerMoveCryovial.test.tsx +++ b/packages/web/src/pages/__tests__/ContainerMoveCryovial.test.tsx @@ -489,4 +489,108 @@ describe('ContainerMoveCryovial', () => { expect(screen.getByText('Resolved Cryovial Tubes')).toBeInTheDocument() }) }) + + describe('a real scanning session', () => { + const box1 = { id: 1, name: 'BOX1', barcode: null, locationId: 1, itemCount: 0, location: { path: '/Loc1' } } + const resolvedBox1 = { + containers: [ + { + identifier: { sourceCollectionName: 'BOX1', sourcePosition: 'A1' }, + container: { + containerId: 2, + currentCollectionId: 1, + currentCollectionName: 'BOX1', + currentCollectionType: 'cryovial_box', + currentPosition: 'A1', + }, + }, + ], + } + + async function uploadBox1() { + vi.mocked(collectionsApi.listCollectionsByType).mockResolvedValue({ collections: [box1] } as any) + vi.mocked(locationsApi.list).mockResolvedValue({ locations: [{ id: 1, name: 'Loc1', path: '/Loc1', parentId: null }] } as any) + const csvContent = 'source_collection_name,source_position,target_position\nBOX1,A1,B1' + const file = new File([csvContent], 'BOX1.csv', { type: 'text/csv' }) + Object.defineProperty(file, 'text', { value: async () => csvContent }) + + const { container } = await renderWithProviders() + await waitFor(() => { + expect((container.querySelector('input[type="file"]') as HTMLInputElement).disabled).toBe(false) + }) + fireEvent.change(container.querySelector('input[type="file"]') as HTMLInputElement, { target: { files: [file] } }) + await waitFor(() => expect(screen.getByText('Next: Resolve Containers')).not.toBeDisabled(), { timeout: 3000 }) + } + + async function reachResolveStep() { + vi.mocked(collectionsApi.resolveContainers).mockResolvedValue(resolvedBox1 as any) + await uploadBox1() + fireEvent.click(screen.getByText('Next: Resolve Containers')) + await waitFor(() => expect(screen.getByText('Resolved Cryovial Tubes')).toBeInTheDocument()) + } + + it('retries a failed lookup with the uploaded files when Next is selected again', async () => { + vi.mocked(collectionsApi.resolveContainers) + .mockRejectedValueOnce(Object.assign(new Error('Request failed with status code 500'), { + response: { status: 500, data: { error: 'Internal server error' } }, + })) + .mockResolvedValueOnce(resolvedBox1 as any) + await uploadBox1() + + fireEvent.click(screen.getByText('Next: Resolve Containers')) + await waitFor(() => { + expect(screen.getByText(/Lookup failed: Internal server error/)).toBeInTheDocument() + }) + expect(screen.queryByText('Resolved Cryovial Tubes')).not.toBeInTheDocument() + + const next = screen.getByText('Next: Resolve Containers') + expect(next).not.toBeDisabled() + fireEvent.click(next) + + await waitFor(() => expect(screen.getByText('Resolved Cryovial Tubes')).toBeInTheDocument()) + expect(collectionsApi.resolveContainers).toHaveBeenCalledTimes(2) + expect(vi.mocked(collectionsApi.resolveContainers).mock.calls[1]).toEqual( + vi.mocked(collectionsApi.resolveContainers).mock.calls[0], + ) + }) + + it('disables Execute Moves while the request is in flight and sends it once', async () => { + let finishMove!: (value: unknown) => void + vi.mocked(collectionsApi.moveContainers).mockImplementation( + () => new Promise((resolve) => { finishMove = resolve }) as any, + ) + await reachResolveStep() + + const execute = screen.getByRole('button', { name: /execute moves/i }) + fireEvent.click(execute) + fireEvent.click(execute) + + await waitFor(() => expect(screen.getByRole('button', { name: /moving/i })).toBeDisabled()) + fireEvent.click(screen.getByRole('button', { name: /moving/i })) + expect(collectionsApi.moveContainers).toHaveBeenCalledTimes(1) + + finishMove({ success: true, moved: 1 }) + await waitFor(() => expect(screen.getByText('Moves Successful')).toBeInTheDocument()) + expect(collectionsApi.moveContainers).toHaveBeenCalledTimes(1) + }) + + it('names the CSV file and row of a failed move', async () => { + vi.mocked(collectionsApi.moveContainers).mockRejectedValue( + Object.assign(new Error('Request failed with status code 400'), { + response: { + status: 400, + data: { error: 'Move failed', moved: 0, errors: [{ row: 1, error: 'Position B01 is occupied' }] }, + }, + }), + ) + await reachResolveStep() + + fireEvent.click(screen.getByRole('button', { name: /execute moves/i })) + + await waitFor(() => expect(screen.getByText('Moves Failed')).toBeInTheDocument()) + const perFile = screen.getByText('Per-File Results:').parentElement! + expect(perFile).toHaveTextContent('BOX1.csv') + expect(perFile).toHaveTextContent('Row 1: Position B01 is occupied') + }) + }) }) From b6f684736c697865f693c8ba063c1de3030c9f8d Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:44:29 -0700 Subject: [PATCH 13/54] fix(move): report and record what moved (#110) - A move that moved anything returns 200 with `moved` and `errors`; a move that moved nothing still returns 400. Applies to Container and Collection moves. - POST /collections/move takes a Collection type per move, so one request (one transaction) covers plates, boxes and bags and all-or-nothing holds across types. The top-level collectionType remains as a default. Error rows are now 1-based (row 0 is request-level), matching Container moves. - Container moves set storage_container.last_updated/updated_by; Collection moves now set updated_by alongside last_updated. - The Collection Move page sends one request, reads results from both 200 and 400 bodies, refreshes queries whenever anything moved, and names the Collection behind each error row. Co-Authored-By: Claude Opus 5.5 --- .../src/lib/__tests__/collection-move.test.ts | 182 +++++++++++++++- .../src/lib/__tests__/container-move.test.ts | 40 ++++ packages/api/src/lib/collection-move.ts | 197 +++++++++--------- .../src/lib/collections/collection-resolve.ts | 9 +- packages/api/src/lib/container-move.ts | 27 ++- .../src/routes/__tests__/collections.test.ts | 146 +++++++++++++ packages/api/src/routes/collections.ts | 46 ++-- .../bulk-operations/container-movement.md | 4 +- .../docs/guides/features/collection-move.md | 6 +- packages/web/src/lib/api/collections.ts | 8 +- packages/web/src/lib/move-result.ts | 34 +++ packages/web/src/pages/CollectionMove.tsx | 143 ++++--------- .../pages/__tests__/CollectionMove.test.tsx | 118 +++++++++-- 13 files changed, 716 insertions(+), 244 deletions(-) create mode 100644 packages/web/src/lib/move-result.ts diff --git a/packages/api/src/lib/__tests__/collection-move.test.ts b/packages/api/src/lib/__tests__/collection-move.test.ts index f9452cf3..6fff7fd9 100644 --- a/packages/api/src/lib/__tests__/collection-move.test.ts +++ b/packages/api/src/lib/__tests__/collection-move.test.ts @@ -1,9 +1,17 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest' import { setupTestDatabase, cleanupTestDatabase } from '../../__tests__/helpers/db-setup' -import { createTestLocation, createTestStorageType, createTestMicronixPlate } from '../../__tests__/helpers/factories' +import { + createTestLocation, + createTestStorageType, + createTestMicronixPlate, + createTestCryovialBox, + createTestBox, + createTestBag, +} from '../../__tests__/helpers/factories' +import { createTestUser } from '../../__tests__/helpers/auth-helpers' import { resolveLocationByPath, executeCollectionMoves } from '../collection-move' import type { Database } from '../../db/client' -import { micronixPlate } from '../../db/schema' +import { micronixPlate, cryovialBox, box, bag } from '../../db/schema' import { eq } from 'drizzle-orm' describe('collection-move', () => { @@ -137,5 +145,175 @@ describe('collection-move', () => { const afterA = await testDb.select().from(micronixPlate).where(eq(micronixPlate.id, plateA.id)).get() expect(afterA?.locationId).toBe(dst.id) }) + + describe('moves across Collection types in one request', () => { + async function setup() { + const st = await createTestStorageType(testDb, { name: 'Freezer' }) + const src = await createTestLocation(testDb, { name: 'Src', storageTypeId: String(st.id), canContainCollections: true }) + const dst = await createTestLocation(testDb, { name: 'Dst', storageTypeId: String(st.id), canContainCollections: true }) + const shelf = await createTestLocation(testDb, { name: 'Shelf', storageTypeId: String(st.id), canContainCollections: false }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: src.id }) + const cryoBox = await createTestCryovialBox(testDb, { name: 'CryoBox1', locationId: src.id }) + const paperBox = await createTestBox(testDb, { name: 'PaperBox1', locationId: src.id }) + const paperBag = await createTestBag(testDb, { name: 'Bag1', locationId: src.id }) + return { src, dst, shelf, plate, cryoBox, paperBox, paperBag } + } + + function locationOf(table: typeof micronixPlate | typeof cryovialBox | typeof box | typeof bag, id: number) { + return testDb.select({ locationId: table.locationId }).from(table).where(eq(table.id, id)).get()?.locationId + } + + it('moves plates, cryovial boxes, boxes and bags together', async () => { + const { src, dst, plate, cryoBox, paperBox, paperBag } = await setup() + + const result = await executeCollectionMoves(testDb, { + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: cryoBox.id }, targetLocationId: dst.id }, + { collectionType: 'box', identifier: { type: 'id', id: paperBox.id }, targetLocationId: dst.id }, + { collectionType: 'bag', identifier: { type: 'id', id: paperBag.id }, targetLocationId: dst.id }, + ], + }) + + expect(result).toEqual({ success: true, moved: 4, errors: undefined }) + expect(locationOf(micronixPlate, plate.id)).toBe(dst.id) + expect(locationOf(cryovialBox, cryoBox.id)).toBe(dst.id) + expect(locationOf(box, paperBox.id)).toBe(dst.id) + expect(locationOf(bag, paperBag.id)).toBe(dst.id) + expect(src.id).not.toBe(dst.id) + }) + + it('all_or_nothing commits no plate when a box in the same request fails', async () => { + const { src, dst, plate, cryoBox } = await setup() + + const result = await executeCollectionMoves(testDb, { + atomicMode: 'all_or_nothing', + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: cryoBox.id }, targetLocationId: src.id }, + ], + }) + + expect(result.success).toBe(false) + expect(result.moved).toBe(0) + expect(result.errors).toEqual([{ row: 2, error: expect.stringContaining('already at location') }]) + expect(locationOf(micronixPlate, plate.id)).toBe(src.id) + expect(locationOf(cryovialBox, cryoBox.id)).toBe(src.id) + }) + + it('best_effort moves the valid types and reports the failing row by its place in the request', async () => { + const { src, dst, plate, cryoBox, paperBox } = await setup() + + const result = await executeCollectionMoves(testDb, { + atomicMode: 'best_effort', + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: 999_999 }, targetLocationId: dst.id }, + { collectionType: 'box', identifier: { type: 'id', id: paperBox.id }, targetLocationId: dst.id }, + ], + }) + + expect(result.success).toBe(false) + expect(result.moved).toBe(2) + expect(result.errors).toEqual([{ row: 2, error: expect.stringContaining('ID 999999') }]) + expect(locationOf(micronixPlate, plate.id)).toBe(dst.id) + expect(locationOf(box, paperBox.id)).toBe(dst.id) + expect(locationOf(cryovialBox, cryoBox.id)).toBe(src.id) + }) + + it('uses the request collectionType for moves that do not name one', async () => { + const { dst, plate, paperBox } = await setup() + + const result = await executeCollectionMoves(testDb, { + collectionType: 'micronix_plate', + moves: [ + { identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'box', identifier: { type: 'id', id: paperBox.id }, targetLocationId: dst.id }, + ], + }) + + expect(result.success).toBe(true) + expect(result.moved).toBe(2) + }) + + it('rejects a move with no Collection type', async () => { + const { dst, plate } = await setup() + + const result = await executeCollectionMoves(testDb, { + moves: [{ identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }], + }) + + expect(result).toEqual({ success: false, moved: 0, errors: [{ row: 1, error: 'collectionType is required' }] }) + }) + + it('rejects the same Collection listed twice', async () => { + const { src, dst, plate } = await setup() + + const result = await executeCollectionMoves(testDb, { + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'micronix_plate', identifier: { type: 'name', name: 'Plate1' }, targetLocationId: dst.id }, + ], + }) + + expect(result.moved).toBe(0) + expect(result.errors).toEqual([{ row: 2, error: expect.stringContaining('more than once') }]) + expect(locationOf(micronixPlate, plate.id)).toBe(src.id) + }) + }) + + describe('audit fields', () => { + it('sets last_updated and updated_by on each moved Collection', async () => { + const user = await createTestUser(testDb, { email: 'mover@test.com', name: 'Mover', password: 'password123' }) + const st = await createTestStorageType(testDb, { name: 'Freezer' }) + const src = await createTestLocation(testDb, { name: 'Src', storageTypeId: String(st.id), canContainCollections: true }) + const dst = await createTestLocation(testDb, { name: 'Dst', storageTypeId: String(st.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(testDb, { name: 'AuditPlate', locationId: src.id }) + const paperBag = await createTestBag(testDb, { name: 'AuditBag', locationId: src.id }) + const old = '2020-01-01T00:00:00.000Z' + await testDb.update(micronixPlate).set({ lastUpdated: old }).where(eq(micronixPlate.id, plate.id)) + await testDb.update(bag).set({ lastUpdated: old }).where(eq(bag.id, paperBag.id)) + + await executeCollectionMoves( + testDb, + { + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'bag', identifier: { type: 'id', id: paperBag.id }, targetLocationId: dst.id }, + ], + }, + { userId: user.id }, + ) + + const afterPlate = testDb.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + const afterBag = testDb.select().from(bag).where(eq(bag.id, paperBag.id)).get() + expect(afterPlate?.updatedBy).toBe(user.id) + expect(afterPlate?.lastUpdated).not.toBe(old) + expect(afterBag?.updatedBy).toBe(user.id) + expect(afterBag?.lastUpdated).not.toBe(old) + }) + + it('leaves a Collection whose move was skipped unchanged', async () => { + const user = await createTestUser(testDb, { email: 'mover@test.com', name: 'Mover', password: 'password123' }) + const st = await createTestStorageType(testDb, { name: 'Freezer' }) + const src = await createTestLocation(testDb, { name: 'Src', storageTypeId: String(st.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(testDb, { name: 'StayPlate', locationId: src.id }) + const old = '2020-01-01T00:00:00.000Z' + await testDb.update(micronixPlate).set({ lastUpdated: old }).where(eq(micronixPlate.id, plate.id)) + + await executeCollectionMoves( + testDb, + { + atomicMode: 'best_effort', + moves: [{ collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: src.id }], + }, + { userId: user.id }, + ) + + const after = testDb.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + expect(after?.updatedBy).toBeNull() + expect(after?.lastUpdated).toBe(old) + }) + }) }) }) diff --git a/packages/api/src/lib/__tests__/container-move.test.ts b/packages/api/src/lib/__tests__/container-move.test.ts index 838d5a2b..6a2256c5 100644 --- a/packages/api/src/lib/__tests__/container-move.test.ts +++ b/packages/api/src/lib/__tests__/container-move.test.ts @@ -26,6 +26,8 @@ import { storageContainer } from '../../db/schema' import { micronixTube, cryovialTube, staticWell, paper } from '../../db/schema' import type { Database } from '../../db/client' import { utcNow } from '../datetime' +import { createTestUser } from '../../__tests__/helpers/auth-helpers' +import { eq } from 'drizzle-orm' describe('container-move', () => { let testDb: Database @@ -751,4 +753,42 @@ describe('container-move', () => { expect(moved.sheetId).toBe(dest.id) }) }) + + describe('audit fields', () => { + it('sets last_updated and updated_by on each moved Container and leaves skipped ones alone', async () => { + const user = await createTestUser(testDb, { email: 'mover@test.com', name: 'Mover', password: 'password123' }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(testDb, { name: 'AuditPlate', locationId: location.id }) + const moved = await createTestStorageContainer(testDb) + const blocked = await createTestStorageContainer(testDb, { specimenId: moved.specimenId, unitId: moved.unitId }) + await testDb.insert(micronixTube).values([ + { id: moved.id, collectionId: plate.id, barcode: 'MT-AUDIT-1', position: 'A01' }, + { id: blocked.id, collectionId: plate.id, barcode: 'MT-AUDIT-2', position: 'B01' }, + ]) + const old = '2020-01-01T00:00:00.000Z' + await testDb.update(storageContainer).set({ lastUpdated: old }) + + const result = await executeMoves( + testDb, + { + atomicMode: 'best_effort', + mappings: [{ fromCollectionName: 'AuditPlate', toCollectionName: 'AuditPlate' }], + moves: [ + { identifier: { type: 'barcode', barcode: 'MT-AUDIT-1' }, targetPosition: 'C01' }, + { identifier: { type: 'barcode', barcode: 'MT-AUDIT-2' }, targetPosition: 'Z99' }, + ], + }, + { userId: user.id }, + ) + + expect(result.moved).toBe(1) + const afterMoved = testDb.select().from(storageContainer).where(eq(storageContainer.id, moved.id)).get() + const afterBlocked = testDb.select().from(storageContainer).where(eq(storageContainer.id, blocked.id)).get() + expect(afterMoved?.updatedBy).toBe(user.id) + expect(afterMoved?.lastUpdated).not.toBe(old) + expect(afterBlocked?.updatedBy).toBeNull() + expect(afterBlocked?.lastUpdated).toBe(old) + }) + }) }) diff --git a/packages/api/src/lib/collection-move.ts b/packages/api/src/lib/collection-move.ts index 3eaf927d..be835119 100644 --- a/packages/api/src/lib/collection-move.ts +++ b/packages/api/src/lib/collection-move.ts @@ -37,27 +37,45 @@ export interface CollectionIdentifier { } export interface CollectionMoveOperation { + /** Type of the Collection to move; falls back to the request's `collectionType`. */ + collectionType?: MoveableCollectionType identifier: CollectionIdentifier targetLocationId: number } +/** + * One move request, run as one transaction. Each move may name its own Collection type, so + * plates, boxes and bags move together and `all_or_nothing` holds across types. + */ export interface CollectionMoveRequest { - collectionType: MoveableCollectionType + /** Default type for moves that do not name one. */ + collectionType?: MoveableCollectionType moves: CollectionMoveOperation[] atomicMode?: 'all_or_nothing' | 'best_effort' } +/** `row` is the 1-based position of the move in the request; row 0 is a request-level error. */ export interface ValidationError { row: number error: string } +/** + * Outcome of a Collection move. `success` is true only when every move ran. A best-effort + * move can move some Collections (`moved > 0`) and report `errors` for the rest; those moves + * are committed. + */ export interface CollectionMoveResult { success: boolean moved: number errors?: ValidationError[] } +/** Who is moving: recorded as each moved Collection's `updated_by`. */ +export interface CollectionMoveActor { + userId: number | null +} + /** * Resolve location by path string * Note: Users should provide locationId directly. This function is only used @@ -458,110 +476,101 @@ async function validateCollectionMove( return { valid: true } } +function describeIdentifier(identifier: CollectionIdentifier): string { + const what = + identifier.type === 'id' + ? `ID ${identifier.id}` + : identifier.type === 'name' + ? `name "${identifier.name}"` + : `barcode "${identifier.barcode}"` + const where = identifier.locationId + ? ` at location ${identifier.locationId}` + : identifier.locationPath + ? ` at ${identifier.locationPath}` + : '' + const hint = + identifier.type === 'name' || identifier.type === 'barcode' + ? ' If multiple collections match, provide locationId or locationPath for disambiguation.' + : '' + return `Collection not found: ${what}${where}.${hint}` +} + /** - * Execute collection moves + * Execute collection moves in one transaction. `all_or_nothing` commits nothing if any move + * fails validation; `best_effort` commits the valid moves and returns errors for the rest. */ export async function executeCollectionMoves( database: Database, - request: CollectionMoveRequest + request: CollectionMoveRequest, + actor: CollectionMoveActor = { userId: null }, ): Promise { - try { - const { moves, collectionType, atomicMode = 'all_or_nothing' } = request - const errors: ValidationError[] = [] - const validMoves: Array<{ collectionInfo: CollectionInfo; targetLocationId: number }> = [] - - // Resolve all collections - for (let i = 0; i < moves.length; i++) { - const move = moves[i] - const collectionInfo = await resolveCollectionByIdentifier(database, move.identifier, collectionType) - - if (!collectionInfo) { - errors.push({ - row: i, - error: `Collection not found: ${move.identifier.type === 'id' ? `ID ${move.identifier.id}` : move.identifier.type === 'name' ? `name "${move.identifier.name}"` : `barcode "${move.identifier.barcode}"`}${move.identifier.locationId ? ` at location ${move.identifier.locationId}` : move.identifier.locationPath ? ` at ${move.identifier.locationPath}` : ''}. ${move.identifier.type === 'name' || move.identifier.type === 'barcode' ? 'If multiple collections match, provide locationId or locationPath for disambiguation.' : ''}`, - }) - continue - } - - // Validate move - const validation = await validateCollectionMove(database, collectionInfo, move.targetLocationId) - if (!validation.valid) { - errors.push({ - row: i, - error: validation.error || 'Invalid move', - }) - continue - } - - validMoves.push({ - collectionInfo, - targetLocationId: move.targetLocationId, - }) + const { moves, atomicMode = 'all_or_nothing' } = request + const errors: ValidationError[] = [] + const validMoves: Array<{ collectionInfo: CollectionInfo; targetLocationId: number }> = [] + const seen = new Set() + + for (const [index, move] of moves.entries()) { + const row = index + 1 + const collectionType = move.collectionType ?? request.collectionType + if (!collectionType) { + errors.push({ row, error: 'collectionType is required' }) + continue + } + const collectionInfo = await resolveCollectionByIdentifier(database, move.identifier, collectionType) + if (!collectionInfo) { + errors.push({ row, error: describeIdentifier(move.identifier) }) + continue } - if (validMoves.length === 0) { - return { success: false, moved: 0, errors } + const key = `${collectionInfo.collectionType}:${collectionInfo.collectionId}` + if (seen.has(key)) { + errors.push({ row, error: `Collection "${collectionInfo.name}" appears more than once in this move` }) + continue } - if (atomicMode === 'all_or_nothing' && errors.length > 0) { - return { success: false, moved: 0, errors } + seen.add(key) + + const validation = await validateCollectionMove(database, collectionInfo, move.targetLocationId) + if (!validation.valid) { + errors.push({ row, error: validation.error || 'Invalid move' }) + continue } - // Execute moves in transaction - database.transaction((tx) => { - const now = utcNow() - - for (const { collectionInfo, targetLocationId } of validMoves) { - switch (collectionInfo.collectionType) { - case 'micronix_plate': - tx.update(micronixPlate) - .set({ - locationId: targetLocationId, - lastUpdated: now, - }) - .where(eq(micronixPlate.id, collectionInfo.collectionId)) - .run() - break - - case 'cryovial_box': - tx.update(cryovialBox) - .set({ - locationId: targetLocationId, - lastUpdated: now, - }) - .where(eq(cryovialBox.id, collectionInfo.collectionId)) - .run() - break - - case 'box': - tx.update(box) - .set({ - locationId: targetLocationId, - lastUpdated: now, - }) - .where(eq(box.id, collectionInfo.collectionId)) - .run() - break - - case 'bag': - tx.update(bag) - .set({ - locationId: targetLocationId, - lastUpdated: now, - }) - .where(eq(bag.id, collectionInfo.collectionId)) - .run() - break - } - } - }) + validMoves.push({ collectionInfo, targetLocationId: move.targetLocationId }) + } - return { - success: errors.length === 0, - moved: validMoves.length, - errors: errors.length > 0 ? errors : undefined, + if (validMoves.length === 0) { + return { success: false, moved: 0, errors: errors.length > 0 ? errors : [{ row: 0, error: 'No moves provided' }] } + } + if (atomicMode === 'all_or_nothing' && errors.length > 0) { + return { success: false, moved: 0, errors } + } + + database.transaction((tx) => { + // A move changes where the Collection is stored, so record when and by whom. + const audit = { lastUpdated: utcNow(), updatedBy: actor.userId } + + for (const { collectionInfo, targetLocationId } of validMoves) { + const set = { locationId: targetLocationId, ...audit } + switch (collectionInfo.collectionType) { + case 'micronix_plate': + tx.update(micronixPlate).set(set).where(eq(micronixPlate.id, collectionInfo.collectionId)).run() + break + case 'cryovial_box': + tx.update(cryovialBox).set(set).where(eq(cryovialBox.id, collectionInfo.collectionId)).run() + break + case 'box': + tx.update(box).set(set).where(eq(box.id, collectionInfo.collectionId)).run() + break + case 'bag': + tx.update(bag).set(set).where(eq(bag.id, collectionInfo.collectionId)).run() + break + } } - } catch (error) { - throw error + }) + + return { + success: errors.length === 0, + moved: validMoves.length, + errors: errors.length > 0 ? errors : undefined, } } - diff --git a/packages/api/src/lib/collections/collection-resolve.ts b/packages/api/src/lib/collections/collection-resolve.ts index 9a796d5e..a98cf2b1 100644 --- a/packages/api/src/lib/collections/collection-resolve.ts +++ b/packages/api/src/lib/collections/collection-resolve.ts @@ -271,11 +271,18 @@ export const moveSheetsBodySchema = z.object({ targetCollectionType: z.enum(['box', 'bag']), }) +const moveableCollectionTypeSchema = z.enum(['micronix_plate', 'cryovial_box', 'box', 'bag']) + +/** + * One request moves Collections of any type in one transaction: each move names its type, + * or inherits the top-level `collectionType`. + */ export const moveCollectionsBodySchema = z.object({ - collectionType: z.enum(['micronix_plate', 'cryovial_box', 'box', 'bag']), + collectionType: moveableCollectionTypeSchema.optional(), atomicMode: z.enum(['all_or_nothing', 'best_effort']).default('all_or_nothing'), moves: z.array( z.object({ + collectionType: moveableCollectionTypeSchema.optional(), identifier: z.union([ z.object({ type: z.literal('id'), id: z.number().int().positive() }), z.object({ diff --git a/packages/api/src/lib/container-move.ts b/packages/api/src/lib/container-move.ts index 2e786cc2..7da3cda6 100644 --- a/packages/api/src/lib/container-move.ts +++ b/packages/api/src/lib/container-move.ts @@ -9,8 +9,10 @@ import { box, bag, sheet, + storageContainer, } from '../db/schema' import { eq, and, isNull } from 'drizzle-orm' +import { utcNow } from './datetime' import { resolveCollection, resolveSheetsByName, type CollectionType } from './collections/collection-resolve' import { checkGridPositionOccupancy } from './container-occupancy' import { normalizePosition } from './normalize-position' @@ -62,12 +64,22 @@ export interface ValidationError { error: string } +/** + * Outcome of a move. `success` is true only when every row moved. A best-effort move can + * move some rows (`moved > 0`) and still report `errors` for the rest; those moves are + * committed. + */ export interface MoveResult { success: boolean moved: number errors?: ValidationError[] } +/** Who is moving: recorded as each moved Container's `updated_by`. */ +export interface MoveActor { + userId: number | null +} + /** Collections whose containers sit at grid positions. */ type GridCollectionType = 'cryovial_box' | 'micronix_plate' @@ -327,7 +339,11 @@ export function inferCollectionTypeFromContainers(containers: ContainerInfo[]): return { valid: true, collectionType: Array.from(collectionTypes)[0] } } -export async function executeMoves(database: Database, request: BatchMoveRequest): Promise { +export async function executeMoves( + database: Database, + request: BatchMoveRequest, + actor: MoveActor = { userId: null }, +): Promise { try { const { mappings, moves, atomicMode = 'all_or_nothing' } = request const errors: ValidationError[] = [] @@ -553,6 +569,15 @@ export async function executeMoves(database: Database, request: BatchMoveRequest break } } + + // A move changes the Container's placement, so record when and by whom. + const now = utcNow() + for (const { info } of finalMoves) { + tx.update(storageContainer) + .set({ lastUpdated: now, updatedBy: actor.userId }) + .where(eq(storageContainer.id, info.containerId)) + .run() + } }) if (errors.length > 0) { diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 67d286fb..20254177 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -19,6 +19,7 @@ import { createTestSheet, createTestStorageContainer, createTestMicronixTube, + createTestCryovialBox, } from '../../__tests__/helpers/factories' import { setScannerConfigurations } from '../../lib/settings' import { eq } from 'drizzle-orm' @@ -37,6 +38,7 @@ import { qpcrExperimentWell, containerDerivation, studySubject, + users, } from '../../db/schema' describe('Collections API', () => { @@ -205,6 +207,55 @@ describe('Collections API', () => { expect(tube?.position).toBe('A01') }) + it('returns 200 with moved and errors when a best-effort move moves some Containers', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'P96', locationId: loc.id }) + const container = await createTestStorageContainer(ctx.db) + await createTestMicronixTube(ctx.db, { containerId: container.id, collectionId: plate.id, barcode: 'MT-1', position: 'A01' }) + + const res = await ctx.request('/api/collections/containers/move', { + method: 'POST', + json: { + atomicMode: 'best_effort', + mappings: [{ fromCollectionName: 'P96', toCollectionName: 'P96' }], + moves: [ + { identifier: { type: 'barcode', barcode: 'MT-1' }, targetPosition: 'C03' }, + { identifier: { type: 'barcode', barcode: 'MISSING' }, targetPosition: 'D04' }, + ], + }, + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ success: false, moved: 1, errors: [{ row: 2, error: 'Container not found' }] }) + const tube = ctx.db.select().from(micronixTube).where(eq(micronixTube.id, container.id)).get() + expect(tube?.position).toBe('C03') + }) + + it('records the logged-in user and time on a moved Container', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'P96', locationId: loc.id }) + const container = await createTestStorageContainer(ctx.db) + await createTestMicronixTube(ctx.db, { containerId: container.id, collectionId: plate.id, barcode: 'MT-1', position: 'A01' }) + const old = '2020-01-01T00:00:00.000Z' + await ctx.db.update(storageContainer).set({ lastUpdated: old }).where(eq(storageContainer.id, container.id)) + + const res = await ctx.request('/api/collections/containers/move', { + method: 'POST', + json: { + mappings: [{ fromCollectionName: 'P96', toCollectionName: 'P96' }], + moves: [{ identifier: { type: 'barcode', barcode: 'MT-1' }, targetPosition: 'B02' }], + }, + }) + + expect(res.status).toBe(200) + const admin = ctx.db.select().from(users).where(eq(users.email, 'admin@test.com')).get() + const after = ctx.db.select().from(storageContainer).where(eq(storageContainer.id, container.id)).get() + expect(after?.updatedBy).toBe(admin?.id) + expect(after?.lastUpdated).not.toBe(old) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/collections/containers/move', { method: 'POST', @@ -214,6 +265,101 @@ describe('Collections API', () => { }) }) + describe('POST /api/collections/move', () => { + async function setupPlateAndBox() { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const src = await createTestLocation(ctx.db, { name: 'Src', storageTypeId: String(storageType.id), canContainCollections: true }) + const dst = await createTestLocation(ctx.db, { name: 'Dst', storageTypeId: String(storageType.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'Plate1', locationId: src.id }) + const cryoBox = await createTestCryovialBox(ctx.db, { name: 'CryoBox1', locationId: src.id }) + return { src, dst, plate, cryoBox } + } + + it('commits nothing across plates and boxes when an all-or-nothing move has a failing box', async () => { + const { src, dst, plate } = await setupPlateAndBox() + + const res = await ctx.request('/api/collections/move', { + method: 'POST', + json: { + atomicMode: 'all_or_nothing', + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: 999_999 }, targetLocationId: dst.id }, + ], + }, + }) + + expect(res.status).toBe(400) + const body = await res.json() + expect(body.moved).toBe(0) + expect(body.errors).toEqual([{ row: 2, error: expect.stringContaining('ID 999999') }]) + const after = ctx.db.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + expect(after?.locationId).toBe(src.id) + }) + + it('returns 200 with moved and errors when a best-effort move moves some Collections', async () => { + const { src, dst, plate, cryoBox } = await setupPlateAndBox() + + const res = await ctx.request('/api/collections/move', { + method: 'POST', + json: { + atomicMode: 'best_effort', + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: cryoBox.id }, targetLocationId: src.id }, + ], + }, + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ + success: false, + moved: 1, + errors: [{ row: 2, error: expect.stringContaining('already at location') }], + }) + expect(ctx.db.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get()?.locationId).toBe(dst.id) + }) + + it('records the logged-in user and time on each moved Collection', async () => { + const { dst, plate, cryoBox } = await setupPlateAndBox() + const old = '2020-01-01T00:00:00.000Z' + await ctx.db.update(micronixPlate).set({ lastUpdated: old }).where(eq(micronixPlate.id, plate.id)) + await ctx.db.update(cryovialBox).set({ lastUpdated: old }).where(eq(cryovialBox.id, cryoBox.id)) + + const res = await ctx.request('/api/collections/move', { + method: 'POST', + json: { + moves: [ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: cryoBox.id }, targetLocationId: dst.id }, + ], + }, + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ success: true, moved: 2 }) + const admin = ctx.db.select().from(users).where(eq(users.email, 'admin@test.com')).get() + const afterPlate = ctx.db.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + const afterBox = ctx.db.select().from(cryovialBox).where(eq(cryovialBox.id, cryoBox.id)).get() + expect(afterPlate?.updatedBy).toBe(admin?.id) + expect(afterPlate?.lastUpdated).not.toBe(old) + expect(afterBox?.updatedBy).toBe(admin?.id) + expect(afterBox?.lastUpdated).not.toBe(old) + }) + + it('returns 400 for a move with no Collection type', async () => { + const { dst, plate } = await setupPlateAndBox() + + const res = await ctx.request('/api/collections/move', { + method: 'POST', + json: { moves: [{ identifier: { type: 'id', id: plate.id }, targetLocationId: dst.id }] }, + }) + + expect(res.status).toBe(400) + expect((await res.json()).errors).toEqual([{ row: 1, error: 'collectionType is required' }]) + }) + }) + describe('POST /api/collections/sheets/move', () => { it('returns a typed 409 when the target box already has a sheet with that name', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) diff --git a/packages/api/src/routes/collections.ts b/packages/api/src/routes/collections.ts index 9f2cebae..4075293b 100644 --- a/packages/api/src/routes/collections.ts +++ b/packages/api/src/routes/collections.ts @@ -1,4 +1,4 @@ -import { Hono } from 'hono' +import { Hono, type Context } from 'hono' import type { Database } from '../db/client' import { executeMoves, type BatchMoveRequest, type ContainerInfo } from '../lib/container-move' import { executeCollectionMoves, type CollectionMoveRequest } from '../lib/collection-move' @@ -55,6 +55,21 @@ function mapCreateCollectionError(error: unknown, c: { json: (body: unknown, sta return null } +/** + * Respond with a move's result. A move that moved anything returns 200, because those moves + * are committed: `success` is false and `errors` lists the rows a best-effort move skipped. + * A move that moved nothing returns 400 with the errors. + */ +function moveResultResponse( + c: Context, + result: { success: boolean; moved: number; errors?: Array<{ row: number; error: string }> }, +) { + if (result.moved === 0 && !result.success) { + return c.json({ error: 'Move operation failed', moved: 0, errors: result.errors }, 400) + } + return c.json({ success: result.success, moved: result.moved, errors: result.errors }) +} + /** * Create collections routes with database injection * @param database - Database instance (required) @@ -265,13 +280,8 @@ export function createCollectionsRoutes(database: Database): Hono { collections.post('/containers/move', memberMiddleware, async (c) => { try { const data = moveContainersBodySchema.parse(await c.req.json()) - const result = await executeMoves(database, data as BatchMoveRequest) - - if (!result.success) { - return c.json({ error: 'Move operation failed', moved: result.moved, errors: result.errors }, 400) - } - - return c.json({ success: true, moved: result.moved }) + const result = await executeMoves(database, data as BatchMoveRequest, { userId: c.get('user')?.id ?? null }) + return moveResultResponse(c, result) } catch (error) { return handleRouteError(error, c) } @@ -298,24 +308,10 @@ export function createCollectionsRoutes(database: Database): Hono { collections.post('/move', memberMiddleware, async (c) => { try { const data = moveCollectionsBodySchema.parse(await c.req.json()) - const moveResult = await executeCollectionMoves(database, data as CollectionMoveRequest) - - if (!moveResult.success) { - return c.json( - { - error: 'Move operation failed', - moved: moveResult.moved, - errors: moveResult.errors, - }, - 400, - ) - } - - return c.json({ - success: true, - moved: moveResult.moved, - errors: moveResult.errors, + const moveResult = await executeCollectionMoves(database, data as CollectionMoveRequest, { + userId: c.get('user')?.id ?? null, }) + return moveResultResponse(c, moveResult) } catch (error: unknown) { return handleRouteError(error, c) } diff --git a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md index b7ae927b..a538c54f 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md @@ -56,9 +56,9 @@ On **Create Plates**, pick a **location** for each new plate and optionally a ** On **Resolve**, review source plates and unresolved barcodes. Fix the CSV on Upload if needed. Choose **atomicity mode**: - **All-or-nothing (default):** any invalid row blocks all moves. -- **Best effort:** valid rows move; invalid rows are returned as errors. +- **Best effort:** valid rows move; invalid rows are returned as errors. The rows that moved stay moved, so fix and resubmit only the listed rows. -Click **Execute Moves**. Use **Back** to return to Upload, or to Create Plates if destinations still need creating. +Click **Execute Moves**. Each moved container records the move as its last update, with your user as the updater. Use **Back** to return to Upload, or to Create Plates if destinations still need creating. ### Multiple files diff --git a/packages/docs/src/content/docs/guides/features/collection-move.md b/packages/docs/src/content/docs/guides/features/collection-move.md index eaa5db8c..d87844da 100644 --- a/packages/docs/src/content/docs/guides/features/collection-move.md +++ b/packages/docs/src/content/docs/guides/features/collection-move.md @@ -25,13 +25,15 @@ The destination must exist and must be allowed to hold collections. Intermediate **Review & Confirm** lists the collections (type and current location) and the destination (storage type and optional description). Choose atomicity: -- **All-or-nothing (default):** for each collection type in the request, any validation error blocks moves for that type. +- **All-or-nothing (default):** any validation error blocks every move, across all collection types. Plates, boxes, and bags move together in one transaction. - **Best effort:** valid moves are applied; invalid rows are returned as errors. Click **Confirm & Move**. ## Complete -The last step shows how many collections moved and any errors. Click **Move More Collections** to start over. +The last step shows how many collections moved and any errors. Each error names the collection that did not move. If a best-effort move moved some collections, the heading reads **Move Partly Completed**: the listed collections stayed where they were, and the rest moved. Don't repeat the whole move; move only the listed collections again. + +Each moved collection records the move as its last update, with your user as the updater. Click **Move More Collections** to start over. If a destination is rejected, confirm the location exists and that it can contain collections. Create it on **Locations** first if needed. diff --git a/packages/web/src/lib/api/collections.ts b/packages/web/src/lib/api/collections.ts index a8482df0..70284bc5 100644 --- a/packages/web/src/lib/api/collections.ts +++ b/packages/web/src/lib/api/collections.ts @@ -196,10 +196,16 @@ export const collectionsApi = { targetCollectionId: number targetCollectionType: 'box' | 'bag' }) => api.post<{ success: boolean; moved: number }>('/collections/sheets/move', data), + /** + * Move Collections of any type in one transaction. Each move names its type, or inherits + * `collectionType`. Resolves to 200 bodies, including a best-effort move that moved some + * Collections (`success: false`, `moved > 0`, `errors`); a move that moved nothing rejects with 400. + */ moveCollections: (data: { - collectionType: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' + collectionType?: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' atomicMode?: 'all_or_nothing' | 'best_effort' moves: Array<{ + collectionType?: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' identifier: | { type: 'id'; id: number } | { type: 'name'; name: string; locationId?: number; locationPath?: string } diff --git a/packages/web/src/lib/move-result.ts b/packages/web/src/lib/move-result.ts new file mode 100644 index 00000000..6630ad64 --- /dev/null +++ b/packages/web/src/lib/move-result.ts @@ -0,0 +1,34 @@ +/** One row a move could not apply. `row` is the 1-based position of the move in the request; 0 is request-level. */ +export interface MoveRowError { + row: number + error: string +} + +/** + * Outcome of a Container or Collection move. `success` is true only when every move ran; a + * best-effort move can report `moved > 0` together with `errors`, and those moves are committed. + */ +export interface MoveResult { + success: boolean + moved: number + errors?: MoveRowError[] +} + +/** + * Read the result of a move request that was rejected. The API answers a move that moved + * nothing with 400 `{ error, moved, errors }`; any other failure (network, 5xx) becomes one + * request-level error with `moved: 0`. + */ +export function moveResultFromRejection(err: unknown, fallback: string): MoveResult { + const data = + err && typeof err === 'object' && 'response' in err + ? (err as { response?: { data?: { error?: string; moved?: number; errors?: MoveRowError[] } } }).response?.data + : undefined + const errors = Array.isArray(data?.errors) && data.errors.length > 0 ? data.errors : undefined + const message = data?.error || (err instanceof Error ? err.message : '') || fallback + return { + success: false, + moved: typeof data?.moved === 'number' ? data.moved : 0, + errors: errors ?? [{ row: 0, error: message }], + } +} diff --git a/packages/web/src/pages/CollectionMove.tsx b/packages/web/src/pages/CollectionMove.tsx index af8cfb15..102ed0d7 100644 --- a/packages/web/src/pages/CollectionMove.tsx +++ b/packages/web/src/pages/CollectionMove.tsx @@ -2,6 +2,7 @@ import { useState, useMemo } from 'react' import { useQueryClient } from '@tanstack/react-query' import { invalidateAfterBulkWrite } from '../lib/query-client' import { collectionsApi } from '../lib/api/collections' +import { moveResultFromRejection, type MoveResult } from '../lib/move-result' import CollectionMoveTreePicker, { type Collection } from '../components/CollectionMoveTreePicker' import LocationTreePicker, { type LocationSelection } from '../components/LocationTreePicker' import WriteGuard from '../components/WriteGuard' @@ -60,11 +61,9 @@ function CollectionMoveContent() { const [targetLocationId, setTargetLocationId] = useState(null) const [targetLocationPath, setTargetLocationPath] = useState('') const [atomicMode, setAtomicMode] = useState('all_or_nothing') - const [moveResult, setMoveResult] = useState<{ - success: boolean - moved: number - errors?: Array<{ row: number; error: string }> - } | null>(null) + const [moveResult, setMoveResult] = useState(null) + /** The Collections in the order they were sent; an error's `row` N is `movedCollections[N - 1]`. */ + const [movedCollections, setMovedCollections] = useState([]) const handleCollectionToggle = (id: number, type: CollectionType) => { const key = toKey({ type, id }) @@ -126,104 +125,41 @@ function CollectionMoveContent() { setMutating(true) setMoveResult(null) - try { - // Group selected collections by type (each key is "type:id") - const collectionsByType = new Map>() - - Array.from(selectedCollectionIds).forEach((key) => { + // One request moves every selected Collection, whatever its type, in one transaction, + // so all-or-nothing holds across plates, boxes and bags. + const toMove = Array.from(selectedCollectionIds) + .map((key) => { const { type, id } = fromKey(key) - const collection = collections.find((c) => c.type === type && c.id === id) - if (collection) { - if (!collectionsByType.has(type)) { - collectionsByType.set(type, []) - } - collectionsByType.get(type)!.push({ id, collection }) - } + return collections.find((c) => c.type === type && c.id === id) }) + .filter((c): c is Collection => c !== undefined) + setMovedCollections(toMove) - // Execute moves for each type - const allErrors: Array<{ row: number; error: string }> = [] - let totalMoved = 0 - let allSuccess = true - - for (const [collectionType, typeCollections] of collectionsByType.entries()) { - const moves = typeCollections.map(({ id }) => ({ - identifier: { - type: 'id' as const, - id, - }, + let result: MoveResult + try { + result = await collectionsApi.moveCollections({ + atomicMode, + moves: toMove.map((c) => ({ + collectionType: c.type, + identifier: { type: 'id' as const, id: c.id }, targetLocationId, - })) - - try { - const response = await collectionsApi.moveCollections({ - collectionType, - atomicMode, - moves, - }) - - if (response.success) { - totalMoved += response.moved - } else { - allSuccess = false - totalMoved += response.moved || 0 - if (response.errors) { - // Adjust row numbers to account for previous moves - const baseRow = allErrors.length - allErrors.push( - ...response.errors.map((err) => ({ - row: err.row + baseRow, - error: err.error, - })) - ) - } - } - } catch (error: any) { - allSuccess = false - const apiErrors = error.response?.data?.errors - if (Array.isArray(apiErrors) && apiErrors.length > 0) { - const baseRow = allErrors.length - apiErrors.forEach((e: { row: number; error: string }) => { - allErrors.push({ row: baseRow + e.row, error: e.error }) - }) - } else { - allErrors.push({ - row: allErrors.length, - error: error.response?.data?.error || error.message || `Failed to move ${collectionType} collections`, - }) - } - // In strict mode, stop after the first failing type-group - if (atomicMode === 'all_or_nothing') { - break - } - } - } - - if (totalMoved > 0) void invalidateAfterBulkWrite(queryClient) - setMoveResult({ - success: allSuccess, - moved: totalMoved, - errors: allErrors.length > 0 ? allErrors : undefined, - }) - setCurrentStep('execute') - } catch (error: any) { - const apiErrors = error.response?.data?.errors - setMoveResult({ - success: false, - moved: 0, - errors: - Array.isArray(apiErrors) && apiErrors.length > 0 - ? apiErrors.map((e: { row: number; error: string }) => ({ row: e.row, error: e.error })) - : [ - { - row: 0, - error: error.response?.data?.error || error.message || 'Failed to move collections', - }, - ], + })), }) - } finally { - setMutating(false) + } catch (error: unknown) { + result = moveResultFromRejection(error, 'Failed to move collections') } + + // A move that moved anything changed Location and Collection pages, even if some rows failed. + if (result.moved > 0) void invalidateAfterBulkWrite(queryClient) + setMoveResult(result) + setCurrentStep('execute') + setMutating(false) + } + + const describeErrorRow = (row: number): string => { + const collection = row > 0 ? movedCollections[row - 1] : undefined + if (!collection) return 'Request' + return `Row ${row} (${collection.name}, ${getCollectionTypeLabel(collection.type)})` } const handleStartOver = () => { @@ -232,6 +168,7 @@ function CollectionMoveContent() { setTargetLocationId(null) setTargetLocationPath('') setMoveResult(null) + setMovedCollections([]) void collectionsQuery.refetch() } @@ -391,7 +328,7 @@ function CollectionMoveContent() {

{atomicMode === 'all_or_nothing' - ? 'For each collection type request, any validation error blocks moves for that type.' + ? 'Any validation error blocks every move, across all collection types.' : 'Valid moves are applied and invalid rows are returned as errors.'}

@@ -508,12 +445,14 @@ function CollectionMoveContent() { d="M6 18L18 6M6 6l12 12" /> -

Move Failed

+

+ {moveResult.moved > 0 ? 'Move Partly Completed' : 'Move Failed'} +

{moveResult.moved > 0 - ? `Moved ${moveResult.moved} collection${moveResult.moved !== 1 ? 's' : ''}, but some errors occurred.` - : 'Failed to move collections.'} + ? `Moved ${moveResult.moved} collection${moveResult.moved !== 1 ? 's' : ''} to ${targetLocationPath || `location ${targetLocationId}`}. The rows below did not move.` + : 'No collections were moved.'}

{moveResult.errors && moveResult.errors.length > 0 && (
@@ -521,7 +460,7 @@ function CollectionMoveContent() {
    {moveResult.errors.map((error, idx) => (
  • - Row {error.row + 1}: {error.error} + {describeErrorRow(error.row)}: {error.error}
    • ))}
diff --git a/packages/web/src/pages/__tests__/CollectionMove.test.tsx b/packages/web/src/pages/__tests__/CollectionMove.test.tsx index d0441168..5c08b686 100644 --- a/packages/web/src/pages/__tests__/CollectionMove.test.tsx +++ b/packages/web/src/pages/__tests__/CollectionMove.test.tsx @@ -1,6 +1,7 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import userEvent from '@testing-library/user-event' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { QueryClient } from '@tanstack/react-query' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import { collectionsApi } from '../../lib/api/collections' import { locationsApi } from '../../lib/api/locations' @@ -109,7 +110,7 @@ describe('CollectionMove', () => { const { default: CollectionMove } = await import('../CollectionMove') const user = userEvent.setup() - render() + await render() await waitFor(() => { expect(screen.queryByText(/loading collections/i)).not.toBeInTheDocument() @@ -163,7 +164,7 @@ describe('CollectionMove', () => { const { default: CollectionMove } = await import('../CollectionMove') const user = userEvent.setup() - render() + await render() await waitFor(() => { expect(screen.queryByText(/loading collections/i)).not.toBeInTheDocument() @@ -185,12 +186,10 @@ describe('CollectionMove', () => { expect(collectionsApi.moveCollections).toHaveBeenCalled() }) - expect(collectionsApi.moveCollections).toHaveBeenCalledWith( - expect.objectContaining({ - collectionType: 'micronix_plate', - atomicMode: 'all_or_nothing', - }) - ) + expect(collectionsApi.moveCollections).toHaveBeenCalledWith({ + atomicMode: 'all_or_nothing', + moves: [{ collectionType: 'micronix_plate', identifier: { type: 'id', id: 11 }, targetLocationId: 2 }], + }) }) it('sends best_effort when selected in collection move controls', async () => { @@ -207,7 +206,7 @@ describe('CollectionMove', () => { const { default: CollectionMove } = await import('../CollectionMove') const user = userEvent.setup() - render() + await render() await waitFor(() => { expect(screen.queryByText(/loading collections/i)).not.toBeInTheDocument() @@ -231,10 +230,101 @@ describe('CollectionMove', () => { }) expect(collectionsApi.moveCollections).toHaveBeenCalledWith( - expect.objectContaining({ - collectionType: 'micronix_plate', - atomicMode: 'best_effort', - }) + expect.objectContaining({ atomicMode: 'best_effort' }) ) }) + + describe('move results', () => { + const locations = [ + { id: 1, name: 'F1', path: 'F1', parentId: null, canContainCollections: true, description: undefined, storageTypeId: null, effectiveStorageTypeName: '-80°C', created: '', lastUpdated: '' }, + { id: 2, name: 'F2', path: 'F2', parentId: null, canContainCollections: true, description: undefined, storageTypeId: null, effectiveStorageTypeName: '-80°C', created: '', lastUpdated: '' }, + ] + const collections = [ + { id: 5, name: 'Plate P', type: 'micronix_plate' as const, itemCount: 0, locationId: 1, location: { id: 1, path: 'F1' }, barcode: null }, + { id: 7, name: 'Box B', type: 'cryovial_box' as const, itemCount: 0, locationId: 1, location: { id: 1, path: 'F1' }, barcode: null }, + ] + + /** Select both Collections, pick F2, choose the atomicity mode and confirm. */ + async function moveBoth(atomicLabel: RegExp) { + vi.mocked(locationsApi.list).mockResolvedValue({ locations }) + vi.mocked(collectionsApi.listAllCollections).mockResolvedValue({ collections }) + const queryClient = new QueryClient({ defaultOptions: { queries: { retry: false, gcTime: 0 } } }) + const invalidate = vi.spyOn(queryClient, 'invalidateQueries') + const { default: CollectionMove } = await import('../CollectionMove') + const user = userEvent.setup() + await render(, { queryClient }) + + await user.click(await screen.findByRole('button', { name: /expand\s+f1/i })) + for (const checkbox of screen.getAllByRole('checkbox').slice(0, 2)) await user.click(checkbox) + await user.click(screen.getByRole('button', { name: /continue with 2 collections/i })) + await user.click(screen.getByRole('button', { name: /select locations/i })) + const selectButtons = await screen.findAllByRole('button', { name: /select/i }) + await user.click(selectButtons[2]) + await user.click(screen.getByRole('radio', { name: atomicLabel })) + invalidate.mockClear() + await user.click(screen.getByRole('button', { name: /confirm & move/i })) + return { invalidate } + } + + it('sends plates and boxes in one request so all-or-nothing covers both types', async () => { + vi.mocked(collectionsApi.moveCollections).mockResolvedValue({ success: true, moved: 2 }) + + await moveBoth(/all-or-nothing/i) + + expect(await screen.findByRole('heading', { name: /move completed successfully/i })).toBeInTheDocument() + expect(collectionsApi.moveCollections).toHaveBeenCalledTimes(1) + const request = vi.mocked(collectionsApi.moveCollections).mock.calls[0][0] + expect(request.atomicMode).toBe('all_or_nothing') + expect(request.moves).toEqual( + expect.arrayContaining([ + { collectionType: 'micronix_plate', identifier: { type: 'id', id: 5 }, targetLocationId: 2 }, + { collectionType: 'cryovial_box', identifier: { type: 'id', id: 7 }, targetLocationId: 2 }, + ]), + ) + expect(request.moves).toHaveLength(2) + }) + + it('shows the moved count and the failing row from a partial best-effort move, and refreshes pages', async () => { + vi.mocked(collectionsApi.moveCollections).mockImplementation(async (data) => { + const boxRow = data.moves.findIndex((m) => m.collectionType === 'cryovial_box') + 1 + return { success: false, moved: 1, errors: [{ row: boxRow, error: 'Collection is already at location 2' }] } + }) + + const { invalidate } = await moveBoth(/best effort/i) + + expect(await screen.findByRole('heading', { name: /move partly completed/i })).toBeInTheDocument() + expect(screen.getByText(/moved 1 collection to f2/i)).toBeInTheDocument() + const errors = screen.getByRole('list') + expect(within(errors).getByText(/\(box b, cryovial boxes\): collection is already at location 2/i)).toBeInTheDocument() + expect(invalidate).toHaveBeenCalled() + }) + + it('shows the errors of a move the API rejected and does not refresh pages', async () => { + vi.mocked(collectionsApi.moveCollections).mockImplementation(async (data) => { + const boxRow = data.moves.findIndex((m) => m.collectionType === 'cryovial_box') + 1 + throw Object.assign(new Error('Request failed with status code 400'), { + response: { + status: 400, + data: { error: 'Move operation failed', moved: 0, errors: [{ row: boxRow, error: 'Collection not found: ID 7.' }] }, + }, + }) + }) + + const { invalidate } = await moveBoth(/all-or-nothing/i) + + expect(await screen.findByRole('heading', { name: /move failed/i })).toBeInTheDocument() + expect(screen.getByText(/no collections were moved/i)).toBeInTheDocument() + expect(screen.getByText(/\(box b, cryovial boxes\): collection not found: id 7\./i)).toBeInTheDocument() + expect(invalidate).not.toHaveBeenCalled() + }) + + it('reports a network failure as a request-level error', async () => { + vi.mocked(collectionsApi.moveCollections).mockRejectedValue(new Error('Network Error')) + + await moveBoth(/all-or-nothing/i) + + expect(await screen.findByRole('heading', { name: /move failed/i })).toBeInTheDocument() + expect(screen.getByText(/request: network error/i)).toBeInTheDocument() + }) + }) }) From ec367ef02f43e3bd1e6a3d154546b162c5e64e14 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:44:30 -0700 Subject: [PATCH 14/54] fix(api): apply created_from/created_to on the studies and subjects lists GET /studies and GET /subjects ignored the created-date filters, so the dashboard's 30-day trend for Studies and Subjects compared the total with itself. Both lists now filter on `created` through a shared dateRangeConditions helper (also used by the specimens list), with created_to covering its whole day. Refs #118 Co-Authored-By: Claude Opus 5.5 --- .../api/src/__tests__/helpers/factories.ts | 14 +++++++--- .../api/src/lib/specimens/specimen-read.ts | 9 ++----- packages/api/src/lib/sql-helpers.ts | 17 +++++++++++- packages/api/src/lib/studies/study-read.ts | 26 +++++++++---------- .../api/src/routes/__tests__/studies.test.ts | 15 +++++++++++ .../api/src/routes/__tests__/subjects.test.ts | 16 ++++++++++++ packages/api/src/routes/studies.ts | 10 ++++++- packages/api/src/routes/subjects.ts | 10 ++++--- 8 files changed, 88 insertions(+), 29 deletions(-) diff --git a/packages/api/src/__tests__/helpers/factories.ts b/packages/api/src/__tests__/helpers/factories.ts index b39f0915..abe595c7 100644 --- a/packages/api/src/__tests__/helpers/factories.ts +++ b/packages/api/src/__tests__/helpers/factories.ts @@ -354,13 +354,16 @@ export async function createTestControlDefinition(db: Database, data?: { /** * Create a test study */ -export async function createTestStudy(db: Database, data: { title: string, shortCode: string, leadPerson?: string }) { +export async function createTestStudy( + db: Database, + data: { title: string, shortCode: string, leadPerson?: string, created?: string }, +) { const now = utcNow() const [result] = await db.insert(study).values({ ...data, isLongitudinal: false, leadPerson: data.leadPerson ?? 'Test Person', - created: now, + created: data.created ?? now, lastUpdated: now, }).returning() return result @@ -369,11 +372,14 @@ export async function createTestStudy(db: Database, data: { title: string, short /** * Create a test study subject */ -export async function createTestStudySubject(db: Database, data: { studyId: number, name: string }) { +export async function createTestStudySubject( + db: Database, + data: { studyId: number, name: string, created?: string }, +) { const now = utcNow() const [result] = await db.insert(studySubject).values({ ...data, - created: now, + created: data.created ?? now, lastUpdated: now, }).returning() return result diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 327b24a9..c47d88f8 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -3,7 +3,7 @@ import { specimen, storageContainer, studySubject, study, specimenType, controlB import { eq, and, or, sql } from 'drizzle-orm' import { validatePage, validateLimit } from '../constants' import { resolveContainerByBarcode } from '../identifier-resolution' -import { dateToUpperBound } from '../sql-helpers' +import { dateRangeConditions } from '../sql-helpers' import { likeContains } from '../sql-like' export type ListSpecimensQuery = { @@ -148,12 +148,7 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer if (collectionDateTo) { conditions.push(sql`${specimen.collectionDate} <= ${collectionDateTo}`) } - if (createdFrom) { - conditions.push(sql`${specimen.created} >= ${createdFrom}`) - } - if (createdTo) { - conditions.push(dateToUpperBound(specimen.created, createdTo)) - } + conditions.push(...dateRangeConditions(specimen.created, createdFrom, createdTo)) if (barcode) { const containerId = await resolveContainerByBarcode(database, barcode) diff --git a/packages/api/src/lib/sql-helpers.ts b/packages/api/src/lib/sql-helpers.ts index 3b8e10fd..f7b5c085 100644 --- a/packages/api/src/lib/sql-helpers.ts +++ b/packages/api/src/lib/sql-helpers.ts @@ -1,4 +1,4 @@ -import { lt, lte } from 'drizzle-orm' +import { gte, lt, lte, type SQL } from 'drizzle-orm' import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' import { shiftCalendarDate } from './datetime' @@ -11,6 +11,21 @@ export function dateToUpperBound(column: SQLiteColumn, dateTo: string) { return next ? lt(column, next) : lte(column, dateTo) } +/** + * Conditions for a `created_from` / `created_to` query pair on a timestamp column. Either + * bound may be absent; `dateTo` includes its whole day (see `dateToUpperBound`). + */ +export function dateRangeConditions( + column: SQLiteColumn, + dateFrom: string | undefined, + dateTo: string | undefined, +): SQL[] { + const conditions: SQL[] = [] + if (dateFrom) conditions.push(gte(column, dateFrom)) + if (dateTo) conditions.push(dateToUpperBound(column, dateTo)) + return conditions +} + /** * Ids per `IN (...)` list when a list is chunked. SQLite caps bound parameters per statement * (32766 today, 999 before 3.32); 500 stays far under either cap, leaving room for the diff --git a/packages/api/src/lib/studies/study-read.ts b/packages/api/src/lib/studies/study-read.ts index 5a9675d7..19dd1598 100644 --- a/packages/api/src/lib/studies/study-read.ts +++ b/packages/api/src/lib/studies/study-read.ts @@ -1,12 +1,16 @@ import type { Database } from '../../db/client' import type { Database as SQLiteDatabase } from 'bun:sqlite' import { study, studySubject } from '../../db/schema' -import { eq, inArray, or, sql } from 'drizzle-orm' -import { SQL_IN_CHUNK_SIZE } from '../sql-helpers' +import { and, eq, inArray, or, sql } from 'drizzle-orm' +import { SQL_IN_CHUNK_SIZE, dateRangeConditions } from '../sql-helpers' import { likeContains } from '../sql-like' export type ListStudiesParams = { search?: string + /** Earliest `created` date (YYYY-MM-DD), inclusive. */ + createdFrom?: string + /** Latest `created` date (YYYY-MM-DD), inclusive of the whole day. */ + createdTo?: string page: number limit: number } @@ -35,21 +39,17 @@ export type GetStudySummariesResult = { /** List studies with optional search and pagination. */ export async function listStudies(database: Database, params: ListStudiesParams): Promise { - const { search, page, limit } = params + const { search, createdFrom, createdTo, page, limit } = params const offset = (page - 1) * limit - let query = database.select().from(study) - let countQuery = database.select({ count: sql`COUNT(*)`.as('count') }).from(study) - - if (search) { - const whereClause = or(likeContains(study.title, search), likeContains(study.shortCode, search)) - query = query.where(whereClause) as any - countQuery = countQuery.where(whereClause) as any - } + const whereClause = and( + search ? or(likeContains(study.title, search), likeContains(study.shortCode, search)) : undefined, + ...dateRangeConditions(study.created, createdFrom, createdTo), + ) const [studiesList, countResult] = await Promise.all([ - query.limit(limit).offset(offset), - countQuery, + database.select().from(study).where(whereClause).limit(limit).offset(offset), + database.select({ count: sql`COUNT(*)`.as('count') }).from(study).where(whereClause), ]) const total = countResult[0]?.count || 0 diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index 7591f8a1..8b600254 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -42,6 +42,21 @@ describe('Studies API (list, get, create, update)', () => { expect(data.pagination).toHaveProperty('totalPages') }) + it('counts only studies created inside created_from and created_to', async () => { + await createTestStudy(ctx.db, { title: 'Before', shortCode: 'B1', created: '2026-08-31 23:59:59' }) + await createTestStudy(ctx.db, { title: 'First day', shortCode: 'F1', created: '2026-09-01 00:00:00' }) + await createTestStudy(ctx.db, { title: 'Last day', shortCode: 'L1', created: '2026-09-30 23:59:59' }) + await createTestStudy(ctx.db, { title: 'After', shortCode: 'A1', created: '2026-10-01 00:00:00' }) + + const res = await ctx.request('/api/studies?created_from=2026-09-01&created_to=2026-09-30&limit=1') + expect(res.status).toBe(200) + const data = (await res.json()) as { pagination: { total: number } } + expect(data.pagination.total).toBe(2) + + const toOnly = await ctx.request('/api/studies?created_to=2026-09-30&limit=1') + expect(((await toOnly.json()) as { pagination: { total: number } }).pagination.total).toBe(3) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/studies', { method: 'GET' }) expect(res.status).toBe(401) diff --git a/packages/api/src/routes/__tests__/subjects.test.ts b/packages/api/src/routes/__tests__/subjects.test.ts index 938e4e22..58ce7f3a 100644 --- a/packages/api/src/routes/__tests__/subjects.test.ts +++ b/packages/api/src/routes/__tests__/subjects.test.ts @@ -40,6 +40,22 @@ describe('Subjects API', () => { expect(Array.isArray(data.subjects)).toBe(true) }) + it('counts only subjects created inside created_from and created_to', async () => { + const s = await createTestStudy(ctx.db, { title: 'Study', shortCode: 'S1' }) + await createTestStudySubject(ctx.db, { studyId: s.id, name: 'before', created: '2026-08-31 23:59:59' }) + await createTestStudySubject(ctx.db, { studyId: s.id, name: 'first', created: '2026-09-01 00:00:00' }) + await createTestStudySubject(ctx.db, { studyId: s.id, name: 'last', created: '2026-09-30 23:59:59' }) + await createTestStudySubject(ctx.db, { studyId: s.id, name: 'after', created: '2026-10-01 00:00:00' }) + + const res = await ctx.request('/api/subjects?created_from=2026-09-01&created_to=2026-09-30&limit=1') + expect(res.status).toBe(200) + const data = (await res.json()) as { pagination: { total: number } } + expect(data.pagination.total).toBe(2) + + const fromOnly = await ctx.request('/api/subjects?created_from=2026-09-01&limit=1') + expect(((await fromOnly.json()) as { pagination: { total: number } }).pagination.total).toBe(3) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/subjects', { method: 'GET' }) expect(res.status).toBe(401) diff --git a/packages/api/src/routes/studies.ts b/packages/api/src/routes/studies.ts index 9566d010..3159bf1e 100644 --- a/packages/api/src/routes/studies.ts +++ b/packages/api/src/routes/studies.ts @@ -60,7 +60,15 @@ studies.get('/', authMiddleware, async (c) => { const page = validatePage(c.req.query('page')) const limit = await validateLimit(database, c.req.query('limit')) - return c.json(await listStudies(database, { search, page, limit })) + return c.json( + await listStudies(database, { + search, + createdFrom: c.req.query('created_from'), + createdTo: c.req.query('created_to'), + page, + limit, + }), + ) } catch (error) { return handleRouteError(error, c) } diff --git a/packages/api/src/routes/subjects.ts b/packages/api/src/routes/subjects.ts index 67dd5444..366ee264 100644 --- a/packages/api/src/routes/subjects.ts +++ b/packages/api/src/routes/subjects.ts @@ -11,6 +11,7 @@ import { } from '../db/schema' import { eq, sql, and, inArray, or, isNull } from 'drizzle-orm' import { validatePage, validateLimit } from '../lib/constants' +import { dateRangeConditions } from '../lib/sql-helpers' import { z } from 'zod' import { resolveStudyByShortCode, resolveSubjectByNameAndStudy, resolveSpecimenTypeByName } from '../lib/identifier-resolution' import { validateSubjectName, validateStudyShortCode, validateSpecimenData, validateCollectionDate, validateContainerTypeForSpecimenType, validateUnitForContainerType } from '../lib/validation' @@ -48,10 +49,13 @@ subjects.get('/', authMiddleware, async (c) => { const page = validatePage(c.req.query('page')) const limit = await validateLimit(dbInstance, c.req.query('limit')) const offset = (page - 1) * limit - + const whereClause = and( + ...dateRangeConditions(studySubject.created, c.req.query('created_from'), c.req.query('created_to')), + ) + const [subjectsList, countResult] = await Promise.all([ - dbInstance.select().from(studySubject).limit(limit).offset(offset), - dbInstance.select({ count: sql`COUNT(*)`.as('count') }).from(studySubject), + dbInstance.select().from(studySubject).where(whereClause).limit(limit).offset(offset), + dbInstance.select({ count: sql`COUNT(*)`.as('count') }).from(studySubject).where(whereClause), ]) const total = countResult[0]?.count || 0 From e8e2168d4e020947ce060a9ce04bc7ddf0fad031 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:44:35 -0700 Subject: [PATCH 15/54] fix(web): make dashboard stat tiles honest - Trends compare records created in the last 30 days with the 30 days before (UTC calendar dates), for Specimens, Studies and Subjects. No percentage is shown when the earlier window is empty, and a decline carries a minus sign. - listTotal rejects on a failed request instead of returning 0. Each count is its own query, so a failed count shows an error with Retry on its tile and the other tiles still load (ADR 0001). - The tile's icon moves into the title row and the value no longer wraps. Metric groups take two columns from lg and three only from 2xl, so a six-digit count fits at 1024-1535px next to the sidebar. - User guide says what the trend compares and how a failed card looks. Closes #118 Co-Authored-By: Claude Opus 5.5 --- .../docs/guides/getting-started/dashboard.md | 2 +- .../components/__tests__/MetricCard.test.tsx | 63 +++++- .../src/components/dashboard/MetricCard.tsx | 125 +++++++---- .../src/hooks/__tests__/useDashboard.test.ts | 67 ++++++ packages/web/src/hooks/useDashboard.ts | 112 +++++----- .../src/lib/api/__tests__/list-total.test.ts | 15 +- packages/web/src/lib/api/list-total.ts | 26 +-- packages/web/src/lib/api/studies.ts | 5 +- packages/web/src/lib/api/subjects.ts | 2 +- packages/web/src/pages/Dashboard.tsx | 197 ++++++++---------- .../src/pages/__tests__/Dashboard.test.tsx | 122 ++++++++++- .../web/src/utils/__tests__/trends.test.ts | 42 ++++ packages/web/src/utils/trends.ts | 48 ++++- 13 files changed, 593 insertions(+), 233 deletions(-) create mode 100644 packages/web/src/hooks/__tests__/useDashboard.test.ts create mode 100644 packages/web/src/utils/__tests__/trends.test.ts diff --git a/packages/docs/src/content/docs/guides/getting-started/dashboard.md b/packages/docs/src/content/docs/guides/getting-started/dashboard.md index 5bd79f76..6991d6db 100644 --- a/packages/docs/src/content/docs/guides/getting-started/dashboard.md +++ b/packages/docs/src/content/docs/guides/getting-started/dashboard.md @@ -34,7 +34,7 @@ Metrics sit in three groups: - **Studies:** **Studies** and **Subjects** - **Storage:** **Locations** -Some cards show a 30-day trend. The section heading **Key metrics** is visually hidden; the group labels are what you read on the page. +**Specimens**, **Studies**, and **Subjects** can show a 30-day trend: the percentage change between the number created in the last 30 days and the number created in the 30 days before. A card shows no trend when nothing was created in the earlier 30 days. If a count fails to load, its card says so and offers **Retry** instead of showing a number. The section heading **Key metrics** is visually hidden; the group labels are what you read on the page. ## Recent Studies and Recent Activity diff --git a/packages/web/src/components/__tests__/MetricCard.test.tsx b/packages/web/src/components/__tests__/MetricCard.test.tsx index 954feaba..65e497d3 100644 --- a/packages/web/src/components/__tests__/MetricCard.test.tsx +++ b/packages/web/src/components/__tests__/MetricCard.test.tsx @@ -1,11 +1,12 @@ -import { describe, it, expect } from 'vitest' +import { describe, it, expect, vi } from 'vitest' +import userEvent from '@testing-library/user-event' import { render, screen } from '../../__tests__/helpers/render' import MetricCard from '../dashboard/MetricCard' describe('MetricCard', () => { it('renders title and value', async () => { await render() - expect(screen.getByText('Studies')).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Studies' })).toBeInTheDocument() expect(screen.getByText('42')).toBeInTheDocument() }) @@ -19,4 +20,62 @@ describe('MetricCard', () => { const link = screen.getByRole('link', { name: /10/i }) expect(link).toHaveAttribute('href', '/studies') }) + + it('keeps a six-digit value on one line with the full tile width, beside no icon', async () => { + await render() + const value = screen.getByText('118,069') + // The value may not wrap or break mid-number... + expect(value).toHaveClass('whitespace-nowrap') + expect(value).not.toHaveClass('break-words') + // ...and its row holds only the value and trend: the icon sits in the title row, so the + // value is not squeezed into the space left beside a 36px icon. + const valueRow = value.parentElement as HTMLElement + expect(valueRow.querySelector('.w-9')).toBeNull() + const titleRow = screen.getByRole('heading', { name: 'Specimens' }).parentElement as HTMLElement + expect(titleRow.querySelector('.w-9')).not.toBeNull() + // No ancestor inside the card clips the value. + let el: HTMLElement | null = value.parentElement + while (el && !el.classList.contains('dashboard-card')) { + expect(el).not.toHaveClass('overflow-hidden') + el = el.parentElement + } + }) + + it('shows a decline with a minus sign and describes what the trend compares', async () => { + await render( + , + ) + expect(screen.getByText('-25%')).toBeInTheDocument() + expect(screen.getByText('Added in the last 30 days')).toHaveClass('sr-only') + }) + + it('shows a loading placeholder and no value while loading', async () => { + await render() + expect(screen.getByRole('status')).toHaveTextContent('Loading Subjects') + expect(screen.queryByText('0')).not.toBeInTheDocument() + }) + + it('shows an error with retry instead of a value when the count failed', async () => { + const user = userEvent.setup() + const onRetry = vi.fn() + await render( + , + ) + expect(screen.getByRole('alert')).toHaveTextContent('Could not load subjects.') + expect(screen.queryByText('0')).not.toBeInTheDocument() + expect(screen.queryByRole('link')).not.toBeInTheDocument() + await user.click(screen.getByRole('button', { name: 'Retry loading Subjects' })) + expect(onRetry).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/web/src/components/dashboard/MetricCard.tsx b/packages/web/src/components/dashboard/MetricCard.tsx index 8a06e282..c31e762b 100644 --- a/packages/web/src/components/dashboard/MetricCard.tsx +++ b/packages/web/src/components/dashboard/MetricCard.tsx @@ -1,13 +1,23 @@ import { Link } from 'react-router-dom' +import { Button } from '../../ui' interface MetricCardProps { title: string - value: number | string + /** Ignored unless `status` is `ready`. */ + value?: number | string + /** `loading` and `error` replace the value; an errored tile never shows a number. */ + status?: 'loading' | 'error' | 'ready' + /** Shown on an errored tile. */ + errorMessage?: string + /** Offers a retry button on an errored tile. */ + onRetry?: () => void linkTo?: string trend?: { value: number label?: string positive?: boolean + /** Says what the trend compares; read by screen readers and shown on hover. */ + description?: string } color?: 'blue' | 'green' | 'purple' | 'orange' | 'indigo' onClick?: () => void @@ -72,10 +82,79 @@ function MetricIcon({ title, colorClass }: { title: string; colorClass: string } } } -export default function MetricCard({ title, value, linkTo, trend, color = 'blue', onClick, index }: MetricCardProps) { +function TrendBadge({ trend }: { trend: NonNullable }) { + const up = trend.positive !== false + const tone = up ? 'text-[rgb(var(--app-trend-up))]' : 'text-[rgb(var(--app-trend-down))]' + return ( +
+ + {up ? ( + + ) : ( + + )} + + + {up ? '+' : '-'} + {trend.value}% + + {trend.label && {trend.label}} + {trend.description && {trend.description}} +
+ ) +} + +/** + * Dashboard stat tile. The icon sits in the title row so the value gets the tile's full + * inner width, and the value never wraps or breaks: a six-digit count such as `118,069` + * needs about 6rem at `text-2xl`. The Dashboard's grid keeps every tile wide enough for that. + */ +export default function MetricCard({ + title, + value, + status = 'ready', + errorMessage = 'Could not load this count.', + onRetry, + linkTo, + trend, + color = 'blue', + onClick, + index, +}: MetricCardProps) { const colorClass = colorClasses[color] - const formattedValue = typeof value === 'number' ? value.toLocaleString() : value + const formattedValue = typeof value === 'number' ? value.toLocaleString() : (value ?? '') const revealClass = index != null && index >= 1 && index <= 8 ? `dashboard-reveal dashboard-reveal-${index}` : '' + const header = ( +
+

{title}

+ +
+ ) + + if (status !== 'ready') { + return ( +
+ {header} + {status === 'loading' ? ( +
+ Loading {title} +
+ ) : ( +
+

{errorMessage}

+ {onRetry && ( + + )} +
+ )} +
+ ) + } const content = (
-
-
-

{title}

-
-

{formattedValue}

- {trend && ( -
- - {trend.positive !== false ? ( - - ) : ( - - )} - - - {trend.positive !== false ? '+' : ''} - {trend.value}% - - {trend.label && ( - {trend.label} - )} -
- )} -
-
- + {header} +
+

+ {formattedValue} +

+ {trend && }
) diff --git a/packages/web/src/hooks/__tests__/useDashboard.test.ts b/packages/web/src/hooks/__tests__/useDashboard.test.ts new file mode 100644 index 00000000..e25e62ad --- /dev/null +++ b/packages/web/src/hooks/__tests__/useDashboard.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { fetchDashboardCount, fetchDashboardTrend } from '../useDashboard' +import { studiesApi } from '../../lib/api/studies' +import { specimensApi } from '../../lib/api/specimens' +import { subjectsApi } from '../../lib/api/subjects' + +vi.mock('../../lib/api/studies', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('studies', { studiesApi: { list: vi.fn() } }) +}) + +vi.mock('../../lib/api/specimens', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('specimens', { specimensApi: { search: vi.fn() } }) +}) + +vi.mock('../../lib/api/subjects', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('subjects', { subjectsApi: { list: vi.fn() } }) +}) + +const NOW = new Date('2026-10-31T12:00:00Z') +const CURRENT = { limit: 1, created_from: '2026-10-02' } +const PREVIOUS = { limit: 1, created_from: '2026-09-02', created_to: '2026-10-01' } + +function body(total: number) { + return { pagination: { total, page: 1, limit: 1, totalPages: total } } as never +} + +describe('fetchDashboardTrend', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('counts studies created in each window, never the all-time total', async () => { + vi.mocked(studiesApi.list).mockResolvedValueOnce(body(3)).mockResolvedValueOnce(body(4)) + await expect(fetchDashboardTrend('studies', NOW)).resolves.toEqual({ current: 3, previous: 4 }) + expect(studiesApi.list).toHaveBeenCalledWith(undefined, CURRENT) + expect(studiesApi.list).toHaveBeenCalledWith(undefined, PREVIOUS) + }) + + it('counts subjects created in each window', async () => { + vi.mocked(subjectsApi.list).mockResolvedValueOnce(body(20)).mockResolvedValueOnce(body(5)) + await expect(fetchDashboardTrend('subjects', NOW)).resolves.toEqual({ current: 20, previous: 5 }) + expect(subjectsApi.list).toHaveBeenCalledWith(CURRENT) + expect(subjectsApi.list).toHaveBeenCalledWith(PREVIOUS) + }) + + it('counts specimens created in each window', async () => { + vi.mocked(specimensApi.search).mockResolvedValueOnce(body(15)).mockResolvedValueOnce(body(10)) + await expect(fetchDashboardTrend('specimens', NOW)).resolves.toEqual({ current: 15, previous: 10 }) + expect(specimensApi.search).toHaveBeenCalledWith(CURRENT) + expect(specimensApi.search).toHaveBeenCalledWith(PREVIOUS) + }) + + it('rejects when either window fails', async () => { + vi.mocked(subjectsApi.list).mockResolvedValueOnce(body(20)).mockRejectedValueOnce(new Error('boom')) + await expect(fetchDashboardTrend('subjects', NOW)).rejects.toThrow('boom') + }) +}) + +describe('fetchDashboardCount', () => { + it('rejects on a failed request instead of resolving to 0', async () => { + vi.mocked(subjectsApi.list).mockRejectedValueOnce(new Error('offline')) + await expect(fetchDashboardCount('subjects')).rejects.toThrow('offline') + }) +}) diff --git a/packages/web/src/hooks/useDashboard.ts b/packages/web/src/hooks/useDashboard.ts index 243dfa02..22f8f814 100644 --- a/packages/web/src/hooks/useDashboard.ts +++ b/packages/web/src/hooks/useDashboard.ts @@ -1,5 +1,6 @@ -import { useQuery, type QueryClient } from '@tanstack/react-query' -import { listTotal } from '../lib/api/list-total' +import { useQueries, useQuery, type QueryClient, type UseQueryResult } from '@tanstack/react-query' +import { listTotal, type ListTotalParams, type PaginatedListBody } from '../lib/api/list-total' +import { trendWindows } from '../utils/trends' import { studiesApi, type Study, type StudySummaryBasic } from '../lib/api/studies' import { specimensApi } from '../lib/api/specimens' import { subjectsApi } from '../lib/api/subjects' @@ -10,13 +11,22 @@ import { statisticsApi, type StatisticsData } from '../lib/api/statistics' import { controlsApi } from '../lib/api/controls' import { qpcrExperimentsApi, type QpcrExperiment } from '../lib/api/qpcr' -export type DashboardStats = { - studies: number - specimens: number - subjects: number - containers: number - locations: number -} +/** A count the dashboard shows on a stat tile. */ +export type DashboardMetric = 'studies' | 'specimens' | 'subjects' | 'containers' | 'locations' + +/** Metrics whose tile shows a 30-day trend. */ +export type DashboardTrendMetric = Extract + +export const DASHBOARD_METRICS: readonly DashboardMetric[] = [ + 'studies', + 'specimens', + 'subjects', + 'containers', + 'locations', +] + +/** Count of records created in each trend window. */ +export type DashboardTrendCounts = { current: number; previous: number } export type DashboardActivityItem = { id: number @@ -35,8 +45,10 @@ export type DashboardSecondaryData = { export const dashboardKeys = { all: ['dashboard'] as const, - critical: () => [...dashboardKeys.all, 'critical'] as const, + counts: () => [...dashboardKeys.all, 'counts'] as const, + count: (metric: DashboardMetric) => [...dashboardKeys.counts(), metric] as const, trends: () => [...dashboardKeys.all, 'trends'] as const, + trend: (metric: DashboardTrendMetric) => [...dashboardKeys.trends(), metric] as const, secondary: () => [...dashboardKeys.all, 'secondary'] as const, qpcr: () => [...dashboardKeys.all, 'qpcr'] as const, summaries: (ids: number[]) => @@ -48,50 +60,50 @@ export function invalidateDashboardQueries(queryClient: QueryClient) { void queryClient.invalidateQueries({ queryKey: dashboardKeys.all }) } -export function useDashboardCritical() { - return useQuery({ - queryKey: dashboardKeys.critical(), - queryFn: async (): Promise<{ stats: DashboardStats; fetchedAt: Date }> => { - const [studies, specimens, subjects, containers, locations] = await Promise.all([ - listTotal((p) => studiesApi.list(undefined, p), 'studies'), - listTotal((p) => specimensApi.search(p), 'specimens'), - listTotal((p) => subjectsApi.list(p), 'subjects'), - listTotal((p) => containersApi.list(p), 'containers'), - listTotal(async (p) => { - const res = await locationsApi.list(1, p.limit) - return res - }, 'locations'), - ]) - return { - stats: { studies, specimens, subjects, containers, locations }, - fetchedAt: new Date(), - } - }, +const fetchMetricPage: Record Promise> = { + studies: (p) => studiesApi.list(undefined, p), + specimens: (p) => specimensApi.search(p), + subjects: (p) => subjectsApi.list(p), + containers: (p) => containersApi.list(p), + locations: (p) => locationsApi.list(1, p.limit), +} + +/** Total of one metric. A failed request rejects; it never reads as 0. */ +export function fetchDashboardCount(metric: DashboardMetric): Promise { + return listTotal(fetchMetricPage[metric], metric) +} + +/** Records of one metric created in the last 30 days and in the 30 days before. */ +export async function fetchDashboardTrend( + metric: DashboardTrendMetric, + now: Date = new Date(), +): Promise { + const windows = trendWindows(now) + const [current, previous] = await Promise.all([ + listTotal(fetchMetricPage[metric], metric, windows.current), + listTotal(fetchMetricPage[metric], metric, windows.previous), + ]) + return { current, previous } +} + +/** One query per stat tile, so each tile loads, fails, and retries on its own. */ +export function useDashboardCounts(): Record> { + const results = useQueries({ + queries: DASHBOARD_METRICS.map((metric) => ({ + queryKey: dashboardKeys.count(metric), + queryFn: () => fetchDashboardCount(metric), + })), }) + return Object.fromEntries(DASHBOARD_METRICS.map((metric, i) => [metric, results[i]])) as Record< + DashboardMetric, + UseQueryResult + > } -export function useDashboardTrendStats(enabled: boolean) { +export function useDashboardTrend(metric: DashboardTrendMetric, enabled: boolean) { return useQuery({ - queryKey: dashboardKeys.trends(), - queryFn: async (): Promise => { - const thirtyDaysAgo = new Date() - thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30) - const createdTo = thirtyDaysAgo.toISOString().split('T')[0] - - const [studies, specimens, subjects] = await Promise.all([ - listTotal((p) => studiesApi.list(undefined, p), 'studies', { created_to: createdTo }), - listTotal((p) => specimensApi.search({ ...p, created_to: createdTo }), 'specimens'), - listTotal((p) => subjectsApi.list(p), 'subjects'), - ]) - - return { - studies, - specimens, - subjects, - containers: 0, - locations: 0, - } - }, + queryKey: dashboardKeys.trend(metric), + queryFn: () => fetchDashboardTrend(metric), enabled, staleTime: 5 * 60 * 1000, }) diff --git a/packages/web/src/lib/api/__tests__/list-total.test.ts b/packages/web/src/lib/api/__tests__/list-total.test.ts index 5617ec73..2a3c9b61 100644 --- a/packages/web/src/lib/api/__tests__/list-total.test.ts +++ b/packages/web/src/lib/api/__tests__/list-total.test.ts @@ -11,6 +11,12 @@ describe('listTotal', () => { expect(fetchPage).toHaveBeenCalledWith({ limit: 1 }) }) + it('passes created-date filters through to the request', async () => { + const fetchPage = vi.fn().mockResolvedValue({ pagination: { total: 3 } }) + await listTotal(fetchPage, 'subjects', { created_from: '2026-09-02', created_to: '2026-10-01' }) + expect(fetchPage).toHaveBeenCalledWith({ limit: 1, created_from: '2026-09-02', created_to: '2026-10-01' }) + }) + it('falls back to array length when pagination is missing', async () => { const fetchPage = vi.fn().mockResolvedValue({ specimens: [{ id: 1 }, { id: 2 }], @@ -18,8 +24,13 @@ describe('listTotal', () => { await expect(listTotal(fetchPage, 'specimens')).resolves.toBe(2) }) - it('returns 0 when the request fails', async () => { + it('rejects when the request fails, instead of returning 0', async () => { const fetchPage = vi.fn().mockRejectedValue(new Error('network')) - await expect(listTotal(fetchPage, 'containers')).resolves.toBe(0) + await expect(listTotal(fetchPage, 'containers')).rejects.toThrow('network') + }) + + it('rejects when the body has no total and no rows array', async () => { + const fetchPage = vi.fn().mockResolvedValue({ unexpected: true }) + await expect(listTotal(fetchPage, 'containers')).rejects.toThrow(/containers/) }) }) diff --git a/packages/web/src/lib/api/list-total.ts b/packages/web/src/lib/api/list-total.ts index 9a6933a3..242e7c10 100644 --- a/packages/web/src/lib/api/list-total.ts +++ b/packages/web/src/lib/api/list-total.ts @@ -1,20 +1,22 @@ -type PaginatedListBody = { +export type PaginatedListBody = { pagination?: { total?: number } [key: string]: unknown } -/** Read total row count from a paginated list endpoint (`limit: 1`). */ +export type ListTotalParams = { limit: number; created_from?: string; created_to?: string } + +/** + * Read the total row count from a paginated list endpoint (`limit: 1`). A failed request + * rejects, so the caller can show an error instead of a count of 0. + */ export async function listTotal( - fetchPage: (params: { limit: number; created_to?: string }) => Promise, + fetchPage: (params: ListTotalParams) => Promise, arrayKey: string, - extraParams?: { created_to?: string }, + extraParams?: Omit, ): Promise { - try { - const res = await fetchPage({ limit: 1, ...extraParams }) - if (res.pagination?.total != null) return res.pagination.total - const arr = res[arrayKey] - return Array.isArray(arr) ? arr.length : 0 - } catch { - return 0 - } + const res = await fetchPage({ limit: 1, ...extraParams }) + if (res.pagination?.total != null) return res.pagination.total + const arr = res[arrayKey] + if (Array.isArray(arr)) return arr.length + throw new Error(`List response has neither pagination.total nor a "${arrayKey}" array`) } diff --git a/packages/web/src/lib/api/studies.ts b/packages/web/src/lib/api/studies.ts index 09a0a218..ab8ca9a7 100644 --- a/packages/web/src/lib/api/studies.ts +++ b/packages/web/src/lib/api/studies.ts @@ -54,7 +54,10 @@ type SubjectsListResponse = { type SummariesResponse = { summaries: StudySummaryBasic[] } export const studiesApi = { - list: (search?: string, params?: { page?: number; limit?: number }) => + list: ( + search?: string, + params?: { page?: number; limit?: number; created_from?: string; created_to?: string }, + ) => api.get('/studies', { params: { search, ...params } }), get: (id: number) => api.get(`/studies/${id}`), getSubjects: (id: number, params?: { page?: number; limit?: number }) => diff --git a/packages/web/src/lib/api/subjects.ts b/packages/web/src/lib/api/subjects.ts index 669f6048..f73b24a5 100644 --- a/packages/web/src/lib/api/subjects.ts +++ b/packages/web/src/lib/api/subjects.ts @@ -74,7 +74,7 @@ type SubjectsListResponse = { } export const subjectsApi = { - list: (params?: { page?: number; limit?: number }) => + list: (params?: { page?: number; limit?: number; created_from?: string; created_to?: string }) => api.get('/subjects', { params }), get: (id: number) => api.get(`/subjects/${id}`), getSummary: (id: number) => api.get(`/subjects/${id}/summary`), diff --git a/packages/web/src/pages/Dashboard.tsx b/packages/web/src/pages/Dashboard.tsx index a4b92c40..f38ac1e5 100644 --- a/packages/web/src/pages/Dashboard.tsx +++ b/packages/web/src/pages/Dashboard.tsx @@ -5,27 +5,43 @@ import MetricCard from '../components/dashboard/MetricCard' import RecentStudies from '../components/dashboard/RecentStudies' import ActivityFeed from '../components/dashboard/ActivityFeed' import SystemInsights from '../components/dashboard/SystemInsights' -import { SkeletonCard } from '../ui' import SearchModal from '../components/SearchModal' import { calculateTrend } from '../utils/trends' import { useUser } from '../contexts/UserContext' import { useFocusSearchOnSlash } from '../hooks/useHotkey' import { - useDashboardCritical, - useDashboardTrendStats, + useDashboardCounts, + useDashboardTrend, useDashboardSecondary, useDashboardStudySummaries, useDashboardQpcr, mergeStudiesWithSummaries, + type DashboardTrendCounts, } from '../hooks/useDashboard' -import { Button, PageError, SectionMessage, getQueryErrorMessage } from '../ui' +import type { UseQueryResult } from '@tanstack/react-query' +import { Button, SectionMessage, SkeletonCard, getQueryErrorMessage } from '../ui' -const EMPTY_STATS = { - studies: 0, - specimens: 0, - subjects: 0, - containers: 0, - locations: 0, +const TREND_DESCRIPTION = 'Added in the last 30 days, compared with the 30 days before' + +/** Props that put one count query on its stat tile: loading, error with retry, or the value. */ +function countTileProps(query: UseQueryResult, title: string) { + if (query.isError) { + return { + status: 'error' as const, + errorMessage: getQueryErrorMessage(query.error, `Could not load ${title.toLowerCase()}.`), + onRetry: () => void query.refetch(), + } + } + if (query.data === undefined) return { status: 'loading' as const } + return { status: 'ready' as const, value: query.data } +} + +/** Trend badge for a tile, or undefined until both windows load or when there is no meaningful change. */ +function trendBadge(counts: DashboardTrendCounts | undefined) { + if (!counts) return undefined + const trend = calculateTrend(counts.current, counts.previous) + if (!trend) return undefined + return { ...trend, label: '30d', description: TREND_DESCRIPTION } } export default function Dashboard() { @@ -36,8 +52,10 @@ export default function Dashboard() { const heroSearchRef = useRef(null) useFocusSearchOnSlash(heroSearchRef) - const criticalQuery = useDashboardCritical() - const trendQuery = useDashboardTrendStats(criticalQuery.isSuccess) + const counts = useDashboardCounts() + const studiesTrend = useDashboardTrend('studies', counts.studies.isSuccess) + const specimensTrend = useDashboardTrend('specimens', counts.specimens.isSuccess) + const subjectsTrend = useDashboardTrend('subjects', counts.subjects.isSuccess) const secondaryQuery = useDashboardSecondary() const qpcrQuery = useDashboardQpcr() @@ -47,8 +65,10 @@ export default function Dashboard() { ) const summariesQuery = useDashboardStudySummaries(studyIds) - const stats = criticalQuery.data?.stats ?? EMPTY_STATS - const previousStats = trendQuery.data ?? null + const countsUpdatedAt = Math.max( + 0, + ...Object.values(counts).map((q) => (q.isSuccess ? q.dataUpdatedAt : 0)), + ) const recentStudies = useMemo( () => mergeStudiesWithSummaries(secondaryQuery.data?.studies ?? [], summariesQuery.data), [secondaryQuery.data?.studies, summariesQuery.data] @@ -58,13 +78,6 @@ export default function Dashboard() { const hasControls = secondaryQuery.data?.hasControls ?? false const recentQpcrExperiments = qpcrQuery.data ?? [] - const specimensTrend = previousStats - ? calculateTrend(stats.specimens, previousStats.specimens) - : null - const subjectsTrend = previousStats - ? calculateTrend(stats.subjects, previousStats.subjects) - : null - const handleSearchSubmit = (e: React.FormEvent) => { e.preventDefault() const form = e.currentTarget @@ -80,10 +93,6 @@ export default function Dashboard() { const formatDataAsOf = (d: Date) => d.toLocaleString(undefined, { dateStyle: 'short', timeStyle: 'short' }) - const metricsErrorMessage = criticalQuery.error - ? getQueryErrorMessage(criticalQuery.error, 'Failed to load dashboard metrics') - : 'Failed to load dashboard metrics' - return (
- {criticalQuery.data?.fetchedAt && ( + {countsUpdatedAt > 0 && (

- Data as of {formatDataAsOf(criticalQuery.data.fetchedAt)} + Data as of {formatDataAsOf(new Date(countsUpdatedAt))}

)} @@ -339,88 +348,66 @@ export default function Dashboard() {

Key metrics

- {criticalQuery.isPending ? ( -
- {Array.from({ length: 5 }).map((_, i) => ( - - ))} -
- ) : criticalQuery.isError ? ( - void criticalQuery.refetch()} - /> - ) : ( -
-
-

Inventory

-
- - -
+ {/* + Each tile needs about 150px of inner width for a six-digit count. Three groups side by + side leave that only from 2xl (1536px): at lg the main column is the viewport minus the + 208px sidebar, so three groups at 1280px would make tiles 157px wide (59px for the + number). Below 2xl the groups take two columns, and below lg (no sidebar) one. + */} +
+
+

Inventory

+
+ +
-
-

Studies

-
- - -
+
+
+

Studies

+
+ +
-
-

Storage

-
- -
+
+
+

Storage

+
+
- )} +
diff --git a/packages/web/src/pages/__tests__/Dashboard.test.tsx b/packages/web/src/pages/__tests__/Dashboard.test.tsx index 91fecd8e..0c719e13 100644 --- a/packages/web/src/pages/__tests__/Dashboard.test.tsx +++ b/packages/web/src/pages/__tests__/Dashboard.test.tsx @@ -1,17 +1,37 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' +import { render, screen, within } from '../../__tests__/helpers/render' import Dashboard from '../Dashboard' +import { studiesApi } from '../../lib/api/studies' +import { specimensApi } from '../../lib/api/specimens' +import { subjectsApi } from '../../lib/api/subjects' +import { containersApi } from '../../lib/api/containers' +import { locationsApi } from '../../lib/api/locations' -vi.mock('../../lib/api/client', async () => { +vi.mock('../../lib/api/studies', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') const { dashboardPageMock } = await import('../../__tests__/helpers/mock-api-templates') - return createMockedDomainModule('client', dashboardPageMock()) + return createMockedDomainModule('studies', dashboardPageMock()) }) -vi.mock('../../lib/api/studies', async () => { +vi.mock('../../lib/api/specimens', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') - const { dashboardPageMock } = await import('../../__tests__/helpers/mock-api-templates') - return createMockedDomainModule('studies', dashboardPageMock()) + return createMockedDomainModule('specimens', { specimensApi: { search: vi.fn() } }) +}) + +vi.mock('../../lib/api/subjects', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('subjects', { subjectsApi: { list: vi.fn() } }) +}) + +vi.mock('../../lib/api/containers', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('containers', { containersApi: { list: vi.fn() } }) +}) + +vi.mock('../../lib/api/locations', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('locations', { locationsApi: { list: vi.fn() } }) }) vi.mock('../../lib/api/search', async () => { @@ -43,16 +63,96 @@ vi.mock('../../contexts/UserContext', async () => { return { ...actual, useUser: () => ({ canWrite: true }) } }) +type CountParams = { limit?: number; created_from?: string; created_to?: string } + +/** A list body whose total depends on which window the request asks for. */ +function totals(all: number, current: number, previous: number) { + return (params?: CountParams) => { + const total = params?.created_to ? previous : params?.created_from ? current : all + return { pagination: { total, page: 1, limit: 1, totalPages: total } } + } +} + +function tile(title: string) { + return screen.getByRole('heading', { level: 2, name: title }).closest('.dashboard-card') as HTMLElement +} + describe('Dashboard', () => { beforeEach(() => { vi.clearAllMocks() + const studies = totals(118, 3, 4) + const specimens = totals(118069, 15, 10) + const subjects = totals(63520, 0, 0) + vi.mocked(studiesApi.list).mockImplementation( + async (_search, params) => ({ studies: [], ...studies(params) }) as never, + ) + vi.mocked(specimensApi.search).mockImplementation( + async (params) => ({ specimens: [], ...specimens(params) }) as never, + ) + vi.mocked(subjectsApi.list).mockImplementation( + async (params) => ({ subjects: [], ...subjects(params) }) as never, + ) + vi.mocked(containersApi.list).mockResolvedValue({ + containers: [], + pagination: { total: 120500, page: 1, limit: 1, totalPages: 120500 }, + } as never) + vi.mocked(locationsApi.list).mockResolvedValue({ + locations: [], + pagination: { total: 42, page: 1, limit: 1, totalPages: 42 }, + } as never) + }) + + it('shows each count in full on its tile', async () => { + await render() + expect(await within(await findTile('Specimens')).findByText('118,069')).toBeInTheDocument() + expect(within(tile('Containers')).getByText('120,500')).toBeInTheDocument() + expect(within(tile('Subjects')).getByText('63,520')).toBeInTheDocument() + expect(within(tile('Studies')).getByText('118')).toBeInTheDocument() + expect(within(tile('Locations')).getByText('42')).toBeInTheDocument() + }) + + it('compares the last 30 days with the 30 days before on each trend', async () => { + await render() + // Specimens: 15 created in the last 30 days, 10 in the 30 before. + expect(await within(await findTile('Specimens')).findByText('+50%')).toBeInTheDocument() + // Studies: 3 against 4. + expect(await within(tile('Studies')).findByText('-25%')).toBeInTheDocument() + // Subjects: nothing created in either window, so no percentage at all. + expect(within(tile('Subjects')).queryByText(/%/)).not.toBeInTheDocument() + + const specimenWindows = vi.mocked(specimensApi.search).mock.calls.map(([p]) => p) + expect(specimenWindows).toContainEqual(expect.objectContaining({ created_from: expect.any(String) })) + expect(specimenWindows).toContainEqual( + expect.objectContaining({ created_from: expect.any(String), created_to: expect.any(String) }), + ) + const subjectWindows = vi.mocked(subjectsApi.list).mock.calls.map(([p]) => p) + expect(subjectWindows).toContainEqual( + expect.objectContaining({ created_from: expect.any(String), created_to: expect.any(String) }), + ) }) - it('eventually shows dashboard content or metrics', async () => { + it('shows an error on a tile whose count fails, not 0, and retries it', async () => { + const user = userEvent.setup() + vi.mocked(subjectsApi.list).mockRejectedValue(new Error('Subjects service unavailable')) + await render() - await vi.waitFor(() => { - const el = screen.queryByRole('main') ?? document.querySelector('[class*="dashboard"]') - expect(el).toBeTruthy() - }, { timeout: 3000 }) + + const subjects = await findTile('Subjects') + expect(await within(subjects).findByRole('alert')).toHaveTextContent('Subjects service unavailable') + expect(within(subjects).queryByText('0')).not.toBeInTheDocument() + // The other tiles still show their counts. + expect(await within(tile('Specimens')).findByText('118,069')).toBeInTheDocument() + + vi.mocked(subjectsApi.list).mockResolvedValue({ + subjects: [], + pagination: { total: 7, page: 1, limit: 1, totalPages: 7 }, + } as never) + await user.click(within(subjects).getByRole('button', { name: 'Retry loading Subjects' })) + expect(await within(tile('Subjects')).findByText('7')).toBeInTheDocument() }) }) + +async function findTile(title: string) { + await screen.findByRole('heading', { level: 2, name: title }) + return tile(title) +} diff --git a/packages/web/src/utils/__tests__/trends.test.ts b/packages/web/src/utils/__tests__/trends.test.ts new file mode 100644 index 00000000..83420fdc --- /dev/null +++ b/packages/web/src/utils/__tests__/trends.test.ts @@ -0,0 +1,42 @@ +import { describe, it, expect } from 'vitest' +import { calculateTrend, trendWindows } from '../trends' + +describe('trendWindows', () => { + it('splits the last 60 days into two 30-day windows that meet without overlapping', () => { + expect(trendWindows(new Date('2026-10-31T15:00:00Z'))).toEqual({ + current: { created_from: '2026-10-02' }, + previous: { created_from: '2026-09-02', created_to: '2026-10-01' }, + }) + }) + + it('uses the UTC date, which is how created timestamps are stored', () => { + // 23:30 UTC on Mar 1 is still Mar 1 in UTC whatever the local zone says. + expect(trendWindows(new Date('2026-03-01T23:30:00Z'))).toEqual({ + current: { created_from: '2026-01-31' }, + previous: { created_from: '2026-01-01', created_to: '2026-01-30' }, + }) + }) +}) + +describe('calculateTrend', () => { + it('reports growth from the previous window to the current one', () => { + expect(calculateTrend(15, 10)).toEqual({ value: 50, positive: true }) + }) + + it('reports a decline as a negative trend', () => { + expect(calculateTrend(3, 4)).toEqual({ value: 25, positive: false }) + }) + + it('treats no change as a flat, non-negative trend', () => { + expect(calculateTrend(5, 5)).toEqual({ value: 0, positive: true }) + }) + + it('rounds to one decimal place', () => { + expect(calculateTrend(2, 3)).toEqual({ value: 33.3, positive: false }) + }) + + it('returns null when the previous window is empty, rather than inventing a percentage', () => { + expect(calculateTrend(12, 0)).toBeNull() + expect(calculateTrend(0, 0)).toBeNull() + }) +}) diff --git a/packages/web/src/utils/trends.ts b/packages/web/src/utils/trends.ts index 36a32d4b..e34d8cb1 100644 --- a/packages/web/src/utils/trends.ts +++ b/packages/web/src/utils/trends.ts @@ -1,23 +1,49 @@ +/** Length of each window a dashboard trend compares. */ +export const TREND_WINDOW_DAYS = 30 + +/** A `created_from` / `created_to` pair, as calendar dates (YYYY-MM-DD, UTC). */ +export type CreatedRange = { created_from: string; created_to?: string } + +export type TrendWindows = { + /** The last 30 days, today included. */ + current: CreatedRange + /** The 30 days before `current`. */ + previous: CreatedRange +} + +function utcDateDaysBefore(now: Date, days: number): string { + const d = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) + d.setUTCDate(d.getUTCDate() - days) + return d.toISOString().slice(0, 10) +} + +/** + * The two windows a dashboard trend compares. Created timestamps are stored in UTC, so the + * windows use UTC calendar dates. Both `created_to` bounds include their whole day. + */ +export function trendWindows(now: Date, days = TREND_WINDOW_DAYS): TrendWindows { + return { + current: { created_from: utcDateDaysBefore(now, days - 1) }, + previous: { + created_from: utcDateDaysBefore(now, 2 * days - 1), + created_to: utcDateDaysBefore(now, days), + }, + } +} + /** - * Calculate trend percentage between two values - * @param current Current value - * @param previous Previous value - * @returns Object with value (percentage change) and positive flag + * Percentage change from `previous` to `current`, rounded to one decimal place. Returns null + * when `previous` is 0: a change from nothing has no meaningful percentage. */ export function calculateTrend(current: number, previous: number): { value: number positive: boolean } | null { - if (previous === 0) { - // If previous is 0, we can't calculate a meaningful trend - if (current === 0) return null - return { value: 100, positive: true } - } + if (previous === 0) return null const change = ((current - previous) / previous) * 100 return { - value: Math.abs(Math.round(change * 10) / 10), // Round to 1 decimal place + value: Math.abs(Math.round(change * 10) / 10), positive: change >= 0, } } - From ddd5e6c179250ad7732249f0e5036d9e4c3ec05f Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:45:25 -0700 Subject: [PATCH 16/54] fix: make destructive actions admin-only per the role policy (#112) Members could delete a Collection with its contents (and run its preflight), delete a Control batch with its Specimens, and edit or delete a Derivation, although the user guide reserves these for admins. These routes now use admin middleware and return 403 to members. The web app hides Delete collection (through a shared CollectionDeleteButton) and Delete Batch from non-admins. The app has no Derivation edit or delete UI. CONTEXT.md now defines the Admin, Member and Viewer roles and lists each Destructive action, and the user guide says Collection and Control batch deletes need an admin. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 26 +++++++++ .../src/routes/__tests__/collections.test.ts | 45 ++++++++++++++++ .../api/src/routes/__tests__/controls.test.ts | 31 ++++++++++- .../src/routes/__tests__/derivations.test.ts | 53 +++++++++++++++++++ packages/api/src/routes/collections.ts | 8 +-- packages/api/src/routes/controls.ts | 7 +-- packages/api/src/routes/derivations.ts | 11 ++-- .../docs/guides/features/blood-controls.md | 4 ++ .../docs/guides/workflows/containers.md | 2 +- .../src/components/CollectionDeleteButton.tsx | 23 ++++++++ .../__tests__/CollectionDeleteButton.test.tsx | 33 ++++++++++++ packages/web/src/pages/BagDetail.tsx | 9 +--- packages/web/src/pages/BoxDetail.tsx | 9 +--- packages/web/src/pages/ControlBatchDetail.tsx | 11 ++-- packages/web/src/pages/CryovialBoxDetail.tsx | 9 +--- .../web/src/pages/MicronixPlateDetail.tsx | 9 +--- .../__tests__/ControlBatchDetail.test.tsx | 17 +++++- 17 files changed, 261 insertions(+), 46 deletions(-) create mode 100644 packages/web/src/components/CollectionDeleteButton.tsx create mode 100644 packages/web/src/components/__tests__/CollectionDeleteButton.test.tsx diff --git a/CONTEXT.md b/CONTEXT.md index 99674c4f..0a1399af 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -14,6 +14,32 @@ _Avoid_: End user, operator (without context) Lab staff responsible for long-term specimen storage, inventory accuracy, and sample retrieval. _Avoid_: Curator (unless that is your org's term) +### Roles + +Every account has one role. The API enforces it on each route, and the web app hides actions the role cannot take. + +**Admin**: +Can do everything a **Member** can, plus manage users, settings, **Reference data**, **Locations**, and every **Destructive action**. + +**Member**: +Lab staff who create and change **Operational data**: register, store, move, derive, and import. A member cannot take a **Destructive action**, except deleting a tutorial **Study** (short code starting with TUT). +_Avoid_: User (every role is a user) + +**Viewer**: +Can read data and run exports, but not create or change anything. + +**Destructive action**: +An action that removes or rewrites recorded **Operational data** or configuration in bulk, or that the app cannot undo. Admin-only. Today these are: + +- Deleting a **Collection** with its contents, and its preflight (removes every **Container** in it, orphaned **Specimens**, and optionally emptied **Subjects**) +- Deleting an empty **Collection** (Data Integrity) +- Deleting a **Control batch** (removes its **Specimens** and their **Containers**) +- Editing or deleting a **Derivation** record +- Deleting a non-tutorial **Study** (removes its **Subjects**, **Specimens**, and **Containers**) +- Deleting **Reference data**, a **Location**, or a user account + +_Avoid_: Calling an ordinary edit (renaming a **Collection**, changing a **Container**'s quantity, removing a **Tag** from a **Container**) a destructive action. + ### Provenance hierarchy **Study → Subject → Specimen** is the primary provenance chain. Most specimens trace back to a study participant or sample source through this path. diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 7a80c107..2c857309 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -48,6 +48,7 @@ describe('Collections API', () => { password: 'password123', role: 'admin', }, + additionalUsers: [{ key: 'member', email: 'member@test.com', password: 'password123', role: 'member' }], mount: (app, { db }) => { app.route('/api/collections', createCollectionsRoutes(db)) }, @@ -719,4 +720,48 @@ describe('Collections API', () => { expect(leftSubj.length).toBe(0) }) }) + describe('delete-with-contents role policy (admin-only)', () => { + async function createEmptyPlate(name: string) { + const storageType = await createTestStorageType(ctx.db, { name: `St-${name}` }) + const loc = await createTestLocation(ctx.db, { + name: `Loc-${name}`, + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + return createTestMicronixPlate(ctx.db, { name, locationId: loc.id }) + } + + it('returns 403 to a member for the preflight', async () => { + const plate = await createEmptyPlate('PlateRolePre') + const res = await ctx.request('/api/collections/delete-with-contents/preflight', { + method: 'POST', + json: { collectionType: 'micronix_plate', id: plate.id }, + cookie: ctx.cookies.member, + }) + expect(res.status).toBe(403) + }) + + it('returns 403 to a member and keeps the Collection', async () => { + const plate = await createEmptyPlate('PlateRoleDel') + const res = await ctx.request('/api/collections/delete-with-contents', { + method: 'POST', + json: { collectionType: 'micronix_plate', id: plate.id, removeEmptySubjects: false }, + cookie: ctx.cookies.member, + }) + expect(res.status).toBe(403) + const still = await ctx.db.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + expect(still).toBeDefined() + }) + + it('lets an admin delete the Collection', async () => { + const plate = await createEmptyPlate('PlateRoleAdmin') + const res = await ctx.request('/api/collections/delete-with-contents', { + method: 'POST', + json: { collectionType: 'micronix_plate', id: plate.id, removeEmptySubjects: false }, + }) + expect(res.status).toBe(200) + const gone = await ctx.db.select().from(micronixPlate).where(eq(micronixPlate.id, plate.id)).get() + expect(gone).toBeUndefined() + }) + }) }) diff --git a/packages/api/src/routes/__tests__/controls.test.ts b/packages/api/src/routes/__tests__/controls.test.ts index fc8a92fb..84542c00 100644 --- a/packages/api/src/routes/__tests__/controls.test.ts +++ b/packages/api/src/routes/__tests__/controls.test.ts @@ -6,9 +6,12 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createControlsRoutes } from '../controls' -import { controlDefinition } from '../../db/schema' +import { controlBatch, controlDefinition, specimen } from '../../db/schema' import { + createTestControlBatch, createTestControlDefinition, + createTestSpecimen, + createTestSpecimenType, createTestStrain, createTestUnit, } from '../../__tests__/helpers/factories' @@ -26,6 +29,7 @@ describe('Controls API', () => { password: 'password123', role: 'admin', }, + additionalUsers: [{ key: 'member', email: 'member@test.com', password: 'password123', role: 'member' }], mount: (app, { db }) => { app.route('/api/blood-controls', createControlsRoutes(db)) }, @@ -641,4 +645,29 @@ describe('Controls API', () => { expect(res.status).toBe(401) }) }) + describe(`DELETE ${BASE}/batches/:id`, () => { + async function createBatchWithSpecimen(name: string) { + const definition = await createTestControlDefinition(ctx.db, { name: `Def-${name}` }) + const batch = await createTestControlBatch(ctx.db, definition.id, { name }) + const specType = await createTestSpecimenType(ctx.db, { name: `Type-${name}` }) + const spec = await createTestSpecimen(ctx.db, specType.id, { controlBatchId: batch.id }) + return { batch, spec } + } + + it('returns 403 to a member and keeps the Control batch and its Specimens', async () => { + const { batch, spec } = await createBatchWithSpecimen('BatchMemberDel') + const res = await ctx.request(`${BASE}/batches/${batch.id}`, { method: 'DELETE', cookie: ctx.cookies.member }) + expect(res.status).toBe(403) + expect(await ctx.db.select().from(controlBatch).where(eq(controlBatch.id, batch.id)).get()).toBeDefined() + expect(await ctx.db.select().from(specimen).where(eq(specimen.id, spec.id)).get()).toBeDefined() + }) + + it('lets an admin delete the Control batch and its Specimens', async () => { + const { batch, spec } = await createBatchWithSpecimen('BatchAdminDel') + const res = await ctx.request(`${BASE}/batches/${batch.id}`, { method: 'DELETE' }) + expect(res.status).toBe(200) + expect(await ctx.db.select().from(controlBatch).where(eq(controlBatch.id, batch.id)).get()).toBeUndefined() + expect(await ctx.db.select().from(specimen).where(eq(specimen.id, spec.id)).get()).toBeUndefined() + }) + }) }) diff --git a/packages/api/src/routes/__tests__/derivations.test.ts b/packages/api/src/routes/__tests__/derivations.test.ts index 47502ec2..b999212b 100644 --- a/packages/api/src/routes/__tests__/derivations.test.ts +++ b/packages/api/src/routes/__tests__/derivations.test.ts @@ -5,6 +5,9 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createDerivationsRoutes } from '../derivations' +import { eq } from 'drizzle-orm' +import { containerDerivation } from '../../db/schema' +import { createTestSpecimen, createTestSpecimenType, createTestStorageContainer } from '../../__tests__/helpers/factories' const BASE = '/api/derivations' @@ -19,6 +22,7 @@ describe('Derivations API', () => { password: 'password123', role: 'admin', }, + additionalUsers: [{ key: 'member', email: 'member@test.com', password: 'password123', role: 'member' }], mount: (app, { db }) => { app.route('/api/derivations', createDerivationsRoutes(db)) }, @@ -113,4 +117,53 @@ describe('Derivations API', () => { expect(res.status).toBe(401) }) }) + describe('PATCH and DELETE /api/derivations/derivations/:id (admin-only)', () => { + async function createDerivationRow() { + const parent = await createTestStorageContainer(ctx.db) + const specType = await createTestSpecimenType(ctx.db, { name: 'DNA' }) + const childSpecimen = await createTestSpecimen(ctx.db, specType.id) + const child = await createTestStorageContainer(ctx.db, { unitId: parent!.unitId, specimenId: childSpecimen!.id }) + const [row] = await ctx.db + .insert(containerDerivation) + .values({ parentContainerId: parent!.id, childContainerId: child!.id, derivationType: 'extraction', protocol: 'P1' }) + .returning() + return row! + } + + async function loadDerivation(id: number) { + return ctx.db.select().from(containerDerivation).where(eq(containerDerivation.id, id)).get() + } + + it('returns 403 to a member editing a Derivation and leaves it unchanged', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { + method: 'PATCH', + json: { protocol: 'P2' }, + cookie: ctx.cookies.member, + }) + expect(res.status).toBe(403) + expect((await loadDerivation(d.id))?.protocol).toBe('P1') + }) + + it('lets an admin edit a Derivation', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'PATCH', json: { protocol: 'P2' } }) + expect(res.status).toBe(200) + expect((await loadDerivation(d.id))?.protocol).toBe('P2') + }) + + it('returns 403 to a member deleting a Derivation and keeps it', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'DELETE', cookie: ctx.cookies.member }) + expect(res.status).toBe(403) + expect(await loadDerivation(d.id)).toBeDefined() + }) + + it('lets an admin delete a Derivation', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'DELETE' }) + expect(res.status).toBe(200) + expect(await loadDerivation(d.id)).toBeUndefined() + }) + }) }) diff --git a/packages/api/src/routes/collections.ts b/packages/api/src/routes/collections.ts index 9f2cebae..b7164bba 100644 --- a/packages/api/src/routes/collections.ts +++ b/packages/api/src/routes/collections.ts @@ -2,7 +2,7 @@ import { Hono } from 'hono' import type { Database } from '../db/client' import { executeMoves, type BatchMoveRequest, type ContainerInfo } from '../lib/container-move' import { executeCollectionMoves, type CollectionMoveRequest } from '../lib/collection-move' -import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' +import { createAdminMiddleware, createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { handleRouteError } from '../lib/error-handler' import { deleteCollectionWithContents, preflightCollectionDelete } from '../lib/collection-delete-cascade' import { validatePlateScan, inferPlateOrGetReport } from '../lib/plate-scan-validation' @@ -63,6 +63,7 @@ export function createCollectionsRoutes(database: Database): Hono { const collections = new Hono() const authMiddleware = createAuthMiddleware(database) const memberMiddleware = createMemberMiddleware(database) + const adminMiddleware = createAdminMiddleware(database) collections.get('/plates/micronix/:id', authMiddleware, async (c) => { const id = parseInt(requireParam(c, 'id')) @@ -321,7 +322,8 @@ export function createCollectionsRoutes(database: Database): Hono { } }) - collections.post('/delete-with-contents/preflight', memberMiddleware, async (c) => { + // Admin-only: deleting a Collection with its contents is a destructive action (CONTEXT.md, Roles). + collections.post('/delete-with-contents/preflight', adminMiddleware, async (c) => { try { const parsed = deleteWithContentsBodySchema .pick({ collectionType: true, id: true }) @@ -336,7 +338,7 @@ export function createCollectionsRoutes(database: Database): Hono { } }) - collections.post('/delete-with-contents', memberMiddleware, async (c) => { + collections.post('/delete-with-contents', adminMiddleware, async (c) => { try { const parsed = deleteWithContentsBodySchema.parse(await c.req.json()) const result = await deleteCollectionWithContents(database, { diff --git a/packages/api/src/routes/controls.ts b/packages/api/src/routes/controls.ts index e9baaf68..bbe69f95 100644 --- a/packages/api/src/routes/controls.ts +++ b/packages/api/src/routes/controls.ts @@ -16,7 +16,7 @@ import { } from '../lib/control-name-generation' import { ConflictError, handleRouteError, ValidationError } from '../lib/error-handler' import type { BloodControlProperties } from '../types/properties' -import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' +import { createAdminMiddleware, createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' import { deleteBloodControlBatch, deleteSpecimenFromBatch } from '../lib/controls/batch-delete' @@ -62,6 +62,7 @@ export function createControlsRoutes(database: Database): Hono { const controls = new Hono() const authMiddleware = createAuthMiddleware(database) const memberMiddleware = createMemberMiddleware(database) + const adminMiddleware = createAdminMiddleware(database) // --- Control Batches --- @@ -141,8 +142,8 @@ controls.patch('/batches/:id', memberMiddleware, async (c) => { } }) -// Delete batch and all associated data -controls.delete('/batches/:id', memberMiddleware, async (c) => { +// Delete batch and all associated data (admin-only: CONTEXT.md, Roles) +controls.delete('/batches/:id', adminMiddleware, async (c) => { const id = parseInt(requireParam(c, 'id')) if (isNaN(id)) return c.json({ error: 'Invalid batch ID' }, 400) diff --git a/packages/api/src/routes/derivations.ts b/packages/api/src/routes/derivations.ts index bdbfff45..f130dfba 100644 --- a/packages/api/src/routes/derivations.ts +++ b/packages/api/src/routes/derivations.ts @@ -10,7 +10,7 @@ import { and, eq } from 'drizzle-orm' import { createDerivation } from '../lib/derivations' import { resolveSpecimenSource } from '../lib/specimens/provenance' import { createDerivationRequestSchema } from '../lib/derivation-schemas' -import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' +import { createAdminMiddleware, createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { requireParam } from '../lib/common-validators' import { handleRouteError } from '../lib/error-handler' import { enrichContainersForApi } from '../lib/container-api-enrichment' @@ -24,6 +24,7 @@ export function createDerivationsRoutes(database: Database): Hono { const derivations = new Hono() const authMiddleware = createAuthMiddleware(database) const memberMiddleware = createMemberMiddleware(database) + const adminMiddleware = createAdminMiddleware(database) // Create derivation from parent container derivations.post('/containers/:id/derive', memberMiddleware, async (c) => { @@ -236,8 +237,8 @@ derivations.get('/containers/:id/derivation-chain', authMiddleware, async (c) => } }) -// Update derivation metadata -derivations.patch('/derivations/:id', memberMiddleware, async (c) => { +// Update derivation metadata (admin-only: CONTEXT.md, Roles) +derivations.patch('/derivations/:id', adminMiddleware, async (c) => { try { const id = parseInt(requireParam(c, 'id')) if (isNaN(id)) { @@ -274,8 +275,8 @@ derivations.patch('/derivations/:id', memberMiddleware, async (c) => { } }) -// Delete derivation (relationship only) -derivations.delete('/derivations/:id', memberMiddleware, async (c) => { +// Delete derivation (relationship only; admin-only: CONTEXT.md, Roles) +derivations.delete('/derivations/:id', adminMiddleware, async (c) => { try { const id = parseInt(requireParam(c, 'id')) if (isNaN(id)) { diff --git a/packages/docs/src/content/docs/guides/features/blood-controls.md b/packages/docs/src/content/docs/guides/features/blood-controls.md index 6b01350c..e6315572 100644 --- a/packages/docs/src/content/docs/guides/features/blood-controls.md +++ b/packages/docs/src/content/docs/guides/features/blood-controls.md @@ -67,6 +67,10 @@ Manual: Batch Info, Specimen Types, CSV Upload, Containers, Review. You can add specimens later from the batch detail page with **Add Specimens**, or import with source type **Control** and the batch name. +## Delete a batch + +Only administrators can delete a batch. On the batch page, click **Delete Batch** and confirm. SampleDB deletes the batch, its specimens, and their containers. This cannot be undone. + ## Use a control as a specimen source On specimen registration, select **Control** as source type, then the batch. Complete specimen type, optional collection date, and container fields as usual. diff --git a/packages/docs/src/content/docs/guides/workflows/containers.md b/packages/docs/src/content/docs/guides/workflows/containers.md index 33841342..e9156e97 100644 --- a/packages/docs/src/content/docs/guides/workflows/containers.md +++ b/packages/docs/src/content/docs/guides/workflows/containers.md @@ -68,7 +68,7 @@ Plate pages include a grid of occupied and empty wells. Box and bag pages list s ## Delete a collection -On a plate, cryovial box, box, or bag (not the sheet-only page), **Delete collection** removes: +Only administrators can delete a collection. On a plate, cryovial box, box, or bag (not the sheet-only page), **Delete collection** removes: - The collection - Every container in it diff --git a/packages/web/src/components/CollectionDeleteButton.tsx b/packages/web/src/components/CollectionDeleteButton.tsx new file mode 100644 index 00000000..a07b79d2 --- /dev/null +++ b/packages/web/src/components/CollectionDeleteButton.tsx @@ -0,0 +1,23 @@ +import { useUser } from '../contexts/UserContext' + +type Props = { + onClick: () => void +} + +/** + * Trigger for deleting a Collection with its contents. Admin-only (CONTEXT.md, Roles), + * so it renders nothing for members and viewers. + */ +export default function CollectionDeleteButton({ onClick }: Props) { + const { isAdmin } = useUser() + if (!isAdmin) return null + return ( + + ) +} diff --git a/packages/web/src/components/__tests__/CollectionDeleteButton.test.tsx b/packages/web/src/components/__tests__/CollectionDeleteButton.test.tsx new file mode 100644 index 00000000..63d85bef --- /dev/null +++ b/packages/web/src/components/__tests__/CollectionDeleteButton.test.tsx @@ -0,0 +1,33 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import userEvent from '@testing-library/user-event' +import { render, screen } from '../../__tests__/helpers/render' +import CollectionDeleteButton from '../CollectionDeleteButton' + +const userState = vi.hoisted(() => ({ isAdmin: false })) + +vi.mock('../../contexts/UserContext', async () => { + const actual = await vi.importActual('../../contexts/UserContext') + return { + ...actual, + useUser: () => ({ isAdmin: userState.isAdmin }), + } +}) + +describe('CollectionDeleteButton', () => { + beforeEach(() => { + userState.isAdmin = false + }) + + it('shows the delete trigger to an admin and calls onClick', async () => { + userState.isAdmin = true + const onClick = vi.fn() + await render() + await userEvent.click(screen.getByRole('button', { name: /Delete collection/i })) + expect(onClick).toHaveBeenCalledTimes(1) + }) + + it('hides the delete trigger from a non-admin (member or viewer)', async () => { + await render() + expect(screen.queryByRole('button', { name: /Delete collection/i })).not.toBeInTheDocument() + }) +}) diff --git a/packages/web/src/pages/BagDetail.tsx b/packages/web/src/pages/BagDetail.tsx index ae2eb7bb..d2d40b04 100644 --- a/packages/web/src/pages/BagDetail.tsx +++ b/packages/web/src/pages/BagDetail.tsx @@ -14,6 +14,7 @@ import { import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { useCollectionSheets } from '../hooks/useCollectionSheets' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' +import CollectionDeleteButton from '../components/CollectionDeleteButton' import '../styles/storage.css' export default function BagDetail() { @@ -133,13 +134,7 @@ export default function BagDetail() { Bag {bag.name}
- + setShowDeleteCollection(true)} />
{bag.locationPath && (

{bag.locationPath}

diff --git a/packages/web/src/pages/BoxDetail.tsx b/packages/web/src/pages/BoxDetail.tsx index cddc425f..74f50861 100644 --- a/packages/web/src/pages/BoxDetail.tsx +++ b/packages/web/src/pages/BoxDetail.tsx @@ -14,6 +14,7 @@ import { import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { useCollectionSheets } from '../hooks/useCollectionSheets' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' +import CollectionDeleteButton from '../components/CollectionDeleteButton' import '../styles/storage.css' export default function BoxDetail() { @@ -136,13 +137,7 @@ export default function BoxDetail() { Box {box.name}
- + setShowDeleteCollection(true)} />
{box.locationPath && (

{box.locationPath}

diff --git a/packages/web/src/pages/ControlBatchDetail.tsx b/packages/web/src/pages/ControlBatchDetail.tsx index 40b6a99a..9b43458e 100644 --- a/packages/web/src/pages/ControlBatchDetail.tsx +++ b/packages/web/src/pages/ControlBatchDetail.tsx @@ -19,7 +19,7 @@ export default function ControlBatchDetail() { const { id } = useParams<{ id: string }>() const navigate = useNavigate() const queryClient = useQueryClient() - const { canWrite } = useUser() + const { canWrite, isAdmin } = useUser() const batchId = id != null ? parseInt(id, 10) : NaN const summaryQuery = useControlBatchSummary(batchId) const summaryStatus = fromQuery(summaryQuery) @@ -279,9 +279,12 @@ export default function ControlBatchDetail() { - + {/* Deleting a Control batch removes its Specimens: admin-only (CONTEXT.md, Roles) */} + {isAdmin && ( + + )}
)}
diff --git a/packages/web/src/pages/CryovialBoxDetail.tsx b/packages/web/src/pages/CryovialBoxDetail.tsx index 81311e3c..96e744cd 100644 --- a/packages/web/src/pages/CryovialBoxDetail.tsx +++ b/packages/web/src/pages/CryovialBoxDetail.tsx @@ -15,6 +15,7 @@ import { import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { Link } from 'react-router-dom' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' +import CollectionDeleteButton from '../components/CollectionDeleteButton' import '../styles/storage.css' function statusColor(name: string): string { @@ -199,13 +200,7 @@ export default function CryovialBoxDetail() { Cryovial Box {box.name}
- + setShowDeleteCollection(true)} /> {box.barcode && (

Barcode: {box.barcode}

diff --git a/packages/web/src/pages/MicronixPlateDetail.tsx b/packages/web/src/pages/MicronixPlateDetail.tsx index 5cd07b2b..41c68f01 100644 --- a/packages/web/src/pages/MicronixPlateDetail.tsx +++ b/packages/web/src/pages/MicronixPlateDetail.tsx @@ -15,6 +15,7 @@ import { import { useTableViewConfigurations } from '../hooks/useTableViewConfigurations' import { Link } from 'react-router-dom' import CollectionDeleteDialog from '../components/CollectionDeleteDialog' +import CollectionDeleteButton from '../components/CollectionDeleteButton' import '../styles/storage.css' function statusColor(name: string): string { @@ -190,13 +191,7 @@ export default function MicronixPlateDetail() { Micronix Plate {plate.name} - + setShowDeleteCollection(true)} /> {plate.barcode && (

Barcode: {plate.barcode}

diff --git a/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx b/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx index 7e3fb284..64edf5c0 100644 --- a/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx +++ b/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx @@ -37,17 +37,20 @@ vi.mock('../../lib/api/controls', async () => { }) }) +const userState = vi.hoisted(() => ({ isAdmin: false })) + vi.mock('../../contexts/UserContext', async () => { const actual = await vi.importActual('../../contexts/UserContext') return { ...actual, - useUser: () => ({ canWrite: true }), + useUser: () => ({ canWrite: true, isAdmin: userState.isAdmin }), } }) describe('ControlBatchDetail', () => { beforeEach(() => { vi.clearAllMocks() + userState.isAdmin = false }) it('shows batch content after load', async () => { @@ -56,4 +59,16 @@ describe('ControlBatchDetail', () => { expect(screen.getByRole('heading', { name: /Batch 1|Control Def|blood/i })).toBeInTheDocument() }, { timeout: 3000 }) }) + + it('shows Delete Batch to an admin', async () => { + userState.isAdmin = true + await render() + expect(await screen.findByRole('button', { name: /Delete Batch/i })).toBeInTheDocument() + }) + + it('hides Delete Batch from a member who can still edit the batch', async () => { + await render() + expect(await screen.findByRole('button', { name: /^Edit$/ })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: /Delete Batch/i })).not.toBeInTheDocument() + }) }) From 971174c9cd79867c09a1890428879a13f69e5b15 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:47:59 -0700 Subject: [PATCH 17/54] fix(position): repair legacy Grid positions and flag invalid ones (#125) Migration 010 brings every stored Grid position into the form the Grid position rule produces: - leading dashes are stripped and the rest normalized (--A01, -F02 -> A01, F02) - a position that is a letter followed by the tube's own barcode is cleared (plate 254, missing_samples) - NA positions are cleared (box 84, NA01-NA05) Cleared Containers stay in their Collection as Unplaced containers. Like 004-006, the migration aborts and changes nothing if a repair would put two Containers in one position of a Collection, or if it finds a position that fits none of these cases. The integrity report gains "Containers with an invalid Grid position": tubes or wells whose stored position fails the rule or is not in stored form. It counts toward the overview's issue total. On the 2026-10-31 production snapshot: 160 dash positions normalized (boxes 790, 794, 795), 201 barcode positions and 5 NA positions cleared, 0 collisions, 0 left that fail the rule; box 790 shows all 73 tubes. Migrations 008 and 009 are on other branches, so this one is numbered 010 and CURRENT_SCHEMA_VERSION jumps to 10; merge 008 and 009 first. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 2 +- packages/api/initial_schema.sql | 2 +- .../src/db/__tests__/schema-evolution.test.ts | 86 +++++++++++++++++++ .../migrations/010_repair_grid_positions.sql | 83 ++++++++++++++++++ packages/api/src/db/schema-evolution.ts | 2 +- packages/api/src/lib/data-audit.ts | 42 ++++++++- .../src/routes/__tests__/data-audit.test.ts | 53 ++++++++++++ .../docs/guides/advanced/data-integrity.md | 1 + packages/web/src/lib/api/admin.ts | 9 ++ .../src/pages/AdminDataIntegrityOverview.tsx | 5 +- .../src/pages/AdminDataIntegrityReport.tsx | 40 +++++++++ ...dminDataIntegrityEmptyCollections.test.tsx | 1 + .../AdminDataIntegrityOverview.test.tsx | 15 ++++ .../AdminDataIntegrityReport.test.tsx | 15 ++++ 14 files changed, 350 insertions(+), 6 deletions(-) create mode 100644 packages/api/src/db/migrations/010_repair_grid_positions.sql diff --git a/CONTEXT.md b/CONTEXT.md index 6b3d4d32..230f8998 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -132,7 +132,7 @@ Where a **Container** sits in the physical storage hierarchy — its **grid posi _Avoid_: Enrichment (implementation term); conflating placement with specimen or source lookups; placing paper directly on a box or bag without a sheet **Grid position**: -The well or slot of a tube or well **Container** in its plate or box, stored as a row letter plus a two-digit column (A01). One rule, the Grid position rule, says which positions a **Collection** has (micronix plate A01–H12, cryovial box A01–J10) and how a typed position maps onto one: `a1`, `A1` and `a01` all mean A01, and anything else, including a cell outside the plate or box, is not a Grid position. Every write that sets a position applies the rule and rejects a position it does not accept; every read that lays Containers out on the grid applies the same rule, so a position a write would reject places no Container (see **Unplaced container**). Legacy rows may still hold malformed positions. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. +The well or slot of a tube or well **Container** in its plate or box, stored as a row letter plus a two-digit column (A01). One rule, the Grid position rule, says which positions a **Collection** has (micronix plate A01–H12, cryovial box A01–J10) and how a typed position maps onto one: `a1`, `A1` and `a01` all mean A01, and anything else, including a cell outside the plate or box, is not a Grid position. Every write that sets a position applies the rule and rejects a position it does not accept; every read that lays Containers out on the grid applies the same rule, so a position a write would reject places no Container (see **Unplaced container**). Legacy malformed positions were repaired by migration 010, and the integrity report flags any stored position that fails the rule. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. _Avoid_: Well (as the domain term); treating a missing position as occupying a slot; requiring every tube to have a position before it can exist in a Collection **Unplaced container**: diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index ce2b56e2..35ea025d 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -284,7 +284,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (7);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (10);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index 65caa5ae..a8e1ecc8 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -311,6 +311,7 @@ describe('schema evolution', () => { const migrated = new Database(testDbPath) migrated.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') migrated.exec('INSERT INTO schema_version (version) VALUES (6)') + createTablesForLaterMigrations(migrated) evolveOperationalSchema(migrated) expect(getRecordedSchemaVersion(migrated)).toBe(CURRENT_SCHEMA_VERSION) expect(qpcrWellResultDdl(migrated)).toContain('CONSTRAINT qpcr_well_result_run_well_target_unique UNIQUE') @@ -329,6 +330,7 @@ describe('schema evolution', () => { UNIQUE (qpcr_run_id, well_position, target_name) )` sqlite.exec(existing) + createTablesForLaterMigrations(sqlite) evolveOperationalSchema(sqlite) @@ -340,6 +342,90 @@ describe('schema evolution', () => { sqlite.close() }) + /** A database at version 9 with grid tables, ready for migration 010. */ + function databaseBeforeMigration010(): Database { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (9)') + createTablesForLaterMigrations(sqlite) + return sqlite + } + + function positions(sqlite: Database, table: string): Array<[number, string | null]> { + const rows = sqlite.prepare(`SELECT id, position FROM ${table} ORDER BY id`).all() as Array<{ id: number; position: string | null }> + return rows.map((r) => [r.id, r.position]) + } + + it('migration 010 strips dashes, clears barcode-shaped and NA positions, and leaves valid ones', () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO cryovial_tube (id, collection_id, barcode, position) VALUES + (1, 790, 'C1', '--A01'), (2, 790, 'C2', '-F02'), (3, 790, 'C3', 'G01'), (4, 84, 'C4', 'NA01'), + (5, 84, 'C5', 'na05'), (6, 84, NULL, NULL), (7, 84, 'C7', 'b3'), (8, 84, 'C8', ' J10 ')`) + sqlite.exec(`INSERT INTO micronix_tube (id, collection_id, barcode, position) VALUES + (11, 254, '1145854300', 'A1145854300'), (12, 254, '1106665111', 'A1106665111'), (13, 254, 'MX3', NULL), (14, 1, 'MX4', 'H12')`) + sqlite.exec(`INSERT INTO static_well (id, collection_id, position) VALUES (21, 1, '-a1')`) + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(positions(sqlite, 'cryovial_tube')).toEqual([ + [1, 'A01'], [2, 'F02'], [3, 'G01'], [4, null], [5, null], [6, null], [7, 'B03'], [8, 'J10'], + ]) + expect(positions(sqlite, 'micronix_tube')).toEqual([[11, null], [12, null], [13, null], [14, 'H12']]) + expect(positions(sqlite, 'static_well')).toEqual([[21, 'A01']]) + const temp = sqlite.prepare(`SELECT name FROM sqlite_temp_master WHERE name LIKE '__migration_010%'`).all() + expect(temp).toEqual([]) + sqlite.close() + }) + + it('migration 010 aborts, changing nothing, when a stripped position collides with a stored one', () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO cryovial_tube (id, collection_id, barcode, position) VALUES (1, 790, 'C1', '--A01'), (2, 790, 'C2', 'A01'), (3, 790, 'C3', '-B01')`) + + expect(() => evolveOperationalSchema(sqlite)).toThrow(/would put two Containers in one position of the same Collection/) + expect(getRecordedSchemaVersion(sqlite)).toBe(9) + expect(positions(sqlite, 'cryovial_tube')).toEqual([[1, '--A01'], [2, 'A01'], [3, '-B01']]) + sqlite.close() + }) + + it('migration 010 aborts when two repairs land in one position', () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO micronix_tube (id, collection_id, barcode, position) VALUES (1, 5, 'M1', '--C04')`) + sqlite.exec(`INSERT INTO static_well (id, collection_id, position) VALUES (2, 5, 'c4')`) + + expect(() => evolveOperationalSchema(sqlite)).toThrow(/would put two Containers in one position/) + expect(getRecordedSchemaVersion(sqlite)).toBe(9) + sqlite.close() + }) + + it('migration 010 allows a collision of the same position in different Collections', () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO cryovial_tube (id, collection_id, barcode, position) VALUES (1, 790, 'C1', '--A01'), (2, 794, 'C2', 'A01')`) + + evolveOperationalSchema(sqlite) + + expect(positions(sqlite, 'cryovial_tube')).toEqual([[1, 'A01'], [2, 'A01']]) + sqlite.close() + }) + + for (const [table, position] of [ + ['cryovial_tube', 'K01'], + ['cryovial_tube', '--A11'], + ['micronix_tube', 'I01'], + ['micronix_tube', 'A001'], + ['micronix_tube', 'hello'], + ] as const) { + it(`migration 010 aborts on a position it cannot repair: ${table} "${position}"`, () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO ${table} (id, collection_id, barcode, position) VALUES (1, 1, 'X1', '--A02'), (2, 1, 'X2', '${position}')`) + + expect(() => evolveOperationalSchema(sqlite)).toThrow(/cannot be repaired automatically/) + expect(getRecordedSchemaVersion(sqlite)).toBe(9) + expect(positions(sqlite, table)).toEqual([[1, '--A02'], [2, position]]) + sqlite.close() + }) + } + it('fail-hard: invalid migration leaves schema_version unchanged', () => { const sqlite = new Database(testDbPath) sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') diff --git a/packages/api/src/db/migrations/010_repair_grid_positions.sql b/packages/api/src/db/migrations/010_repair_grid_positions.sql new file mode 100644 index 00000000..53c7f346 --- /dev/null +++ b/packages/api/src/db/migrations/010_repair_grid_positions.sql @@ -0,0 +1,83 @@ +-- Migration 010: every stored Grid position passes the Grid position rule (issue #125). +-- The rule is `parseGridPosition` in packages/contract/src/grid-position.ts: a row letter and +-- a one- or two-digit column inside the Collection's geometry (micronix plate A01–H12, cryovial +-- box A01–J10), stored as an upper-case letter and a two-digit column. For each position not +-- already in stored form: +-- - leading dashes are stripped and the rest stored in stored form (`--A01`, `-F02`, `a1`) +-- - a position that is a letter followed by the tube's own barcode is cleared (`A1145854300`) +-- - an `NA` position is cleared (`NA`, `NA01`) +-- A cleared Container stays in its Collection as an Unplaced container. Aborts, changing +-- nothing, if any other position is found or if a repair would put two Containers in one +-- position of the same Collection (micronix tubes and static wells share a plate). +CREATE TEMP TABLE __migration_010_repair AS +WITH stored AS ( + SELECT 'micronix_tube' AS tbl, id, barcode, position, 8 AS max_rows, 12 AS max_cols + FROM micronix_tube WHERE position IS NOT NULL + UNION ALL + SELECT 'cryovial_tube', id, barcode, position, 10, 10 + FROM cryovial_tube WHERE position IS NOT NULL + UNION ALL + SELECT 'static_well', id, NULL, position, 8, 12 + FROM static_well WHERE position IS NOT NULL +), +stripped AS ( + SELECT *, ltrim(trim(position), '-') AS s FROM stored +), +parsed AS ( + SELECT *, + CASE + WHEN (s GLOB '[A-Za-z][0-9]' OR s GLOB '[A-Za-z][0-9][0-9]') + AND unicode(upper(substr(s, 1, 1))) - 64 <= max_rows + AND CAST(substr(s, 2) AS INTEGER) BETWEEN 1 AND max_cols + THEN upper(substr(s, 1, 1)) || printf('%02d', CAST(substr(s, 2) AS INTEGER)) + END AS grid + FROM stripped +) +SELECT tbl, id, position AS old_position, grid AS new_position, + CASE + WHEN grid IS NOT NULL THEN 'normalize' + WHEN barcode IS NOT NULL AND barcode <> '' AND s GLOB '[A-Za-z]?*' AND substr(s, 2) = barcode THEN 'clear_barcode' + WHEN upper(s) = 'NA' OR upper(s) GLOB 'NA[0-9]' OR upper(s) GLOB 'NA[0-9][0-9]' THEN 'clear_na' + ELSE 'unexpected' + END AS repair +FROM parsed +WHERE grid IS NULL OR grid <> position;--> statement-breakpoint +CREATE TEMP TABLE __migration_010_final AS +SELECT t.grid_type, t.collection_id, + CASE WHEN r.id IS NULL THEN t.position ELSE r.new_position END AS position, + r.id IS NOT NULL AS repaired +FROM ( + SELECT 'micronix_tube' AS tbl, 'micronix_plate' AS grid_type, id, collection_id, position FROM micronix_tube + UNION ALL SELECT 'cryovial_tube', 'cryovial_box', id, collection_id, position FROM cryovial_tube + UNION ALL SELECT 'static_well', 'micronix_plate', id, collection_id, position FROM static_well +) t +LEFT JOIN __migration_010_repair r ON r.tbl = t.tbl AND r.id = t.id;--> statement-breakpoint +CREATE TEMP TABLE IF NOT EXISTS __migration_010_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint +CREATE TRIGGER __migration_010_unexpected_tr BEFORE INSERT ON __migration_010_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot repair Grid positions: a stored position is not dash-prefixed, the tube''s barcode or NA, so it cannot be repaired automatically. Correct or clear it by hand first.') + WHERE EXISTS (SELECT 1 FROM __migration_010_repair WHERE repair = 'unexpected'); +END;--> statement-breakpoint +CREATE TRIGGER __migration_010_collision_tr BEFORE INSERT ON __migration_010_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot repair Grid positions: a repaired position would put two Containers in one position of the same Collection. Move or clear one of them first.') + WHERE EXISTS ( + SELECT 1 FROM __migration_010_final WHERE position IS NOT NULL + GROUP BY grid_type, collection_id, position HAVING COUNT(*) > 1 AND MAX(repaired) = 1 + ); +END;--> statement-breakpoint +INSERT INTO __migration_010_guard DEFAULT VALUES;--> statement-breakpoint +DROP TRIGGER __migration_010_unexpected_tr;--> statement-breakpoint +DROP TRIGGER __migration_010_collision_tr;--> statement-breakpoint +DROP TABLE __migration_010_guard;--> statement-breakpoint +UPDATE micronix_tube +SET position = (SELECT r.new_position FROM __migration_010_repair r WHERE r.tbl = 'micronix_tube' AND r.id = micronix_tube.id) +WHERE id IN (SELECT id FROM __migration_010_repair WHERE tbl = 'micronix_tube');--> statement-breakpoint +UPDATE cryovial_tube +SET position = (SELECT r.new_position FROM __migration_010_repair r WHERE r.tbl = 'cryovial_tube' AND r.id = cryovial_tube.id) +WHERE id IN (SELECT id FROM __migration_010_repair WHERE tbl = 'cryovial_tube');--> statement-breakpoint +UPDATE static_well +SET position = (SELECT r.new_position FROM __migration_010_repair r WHERE r.tbl = 'static_well' AND r.id = static_well.id) +WHERE id IN (SELECT id FROM __migration_010_repair WHERE tbl = 'static_well');--> statement-breakpoint +DROP TABLE __migration_010_final;--> statement-breakpoint +DROP TABLE __migration_010_repair; diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index b96368bd..a9283ee2 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -7,7 +7,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 7 +export const CURRENT_SCHEMA_VERSION = 10 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/lib/data-audit.ts b/packages/api/src/lib/data-audit.ts index 6311008c..eb49951a 100644 --- a/packages/api/src/lib/data-audit.ts +++ b/packages/api/src/lib/data-audit.ts @@ -19,7 +19,8 @@ import { paper, tag, } from '../db/schema' -import { eq, inArray, sql } from 'drizzle-orm' +import { eq, inArray, isNotNull, sql } from 'drizzle-orm' +import { isStoredGridPosition, type GridCollectionType } from '@sampledb/contract' export type CollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' @@ -341,6 +342,14 @@ export interface ContainerWithNoGridPositionItem { collectionId: number } +export interface ContainerWithInvalidGridPositionItem { + id: number + containerType: 'micronix_tube' | 'cryovial_tube' | 'static_well' + collectionId: number + /** The stored position, as stored. */ + position: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -354,6 +363,7 @@ export interface IntegrityReport { duplicateBarcodes: DuplicateBarcodeItem[] locationPathInconsistencies: LocationPathInconsistencyItem[] containersWithNoGridPosition: ContainerWithNoGridPositionItem[] + containersWithInvalidGridPosition: ContainerWithInvalidGridPositionItem[] } /** Collections (plates, boxes, bags) whose locationId does not exist in location. */ @@ -520,6 +530,33 @@ export async function listContainersWithNoGridPosition(db: Database): Promise { + const sources = [ + { containerType: 'micronix_tube', gridType: 'micronix_plate', table: micronixTube }, + { containerType: 'cryovial_tube', gridType: 'cryovial_box', table: cryovialTube }, + { containerType: 'static_well', gridType: 'micronix_plate', table: staticWell }, + ] as const satisfies ReadonlyArray<{ containerType: ContainerWithInvalidGridPositionItem['containerType']; gridType: GridCollectionType; table: unknown }> + const results: ContainerWithInvalidGridPositionItem[] = [] + for (const { containerType, gridType, table } of sources) { + const rows = await db + .select({ id: table.id, collectionId: table.collectionId, position: table.position }) + .from(table) + .where(isNotNull(table.position)) + for (const row of rows) { + if (row.position != null && !isStoredGridPosition(row.position, gridType)) { + results.push({ id: row.id, containerType, collectionId: row.collectionId, position: row.position }) + } + } + } + return results +} + /** Build full integrity report (all checks). */ export async function getIntegrityReport(db: Database): Promise { const [ @@ -535,6 +572,7 @@ export async function getIntegrityReport(db: Database): Promise duplicateBarcodes, locationPathInconsistencies, containersWithNoGridPosition, + containersWithInvalidGridPosition, ] = await Promise.all([ listEmptyCollections(db), listCollectionsWithMissingLocation(db), @@ -548,6 +586,7 @@ export async function getIntegrityReport(db: Database): Promise listDuplicateBarcodes(db), listLocationPathInconsistencies(db), listContainersWithNoGridPosition(db), + listContainersWithInvalidGridPosition(db), ]) return { emptyCollections, @@ -562,5 +601,6 @@ export async function getIntegrityReport(db: Database): Promise duplicateBarcodes, locationPathInconsistencies, containersWithNoGridPosition, + containersWithInvalidGridPosition, } } diff --git a/packages/api/src/routes/__tests__/data-audit.test.ts b/packages/api/src/routes/__tests__/data-audit.test.ts index 628626f3..7bf51b68 100644 --- a/packages/api/src/routes/__tests__/data-audit.test.ts +++ b/packages/api/src/routes/__tests__/data-audit.test.ts @@ -18,6 +18,9 @@ import { createTestControlDefinition, createTestUnit, createTestTag, + createTestCryovialBox, + createTestCryovialTube, + createTestMicronixTube, } from '../../__tests__/helpers/factories' import { micronixTube, @@ -245,6 +248,7 @@ describe('Data audit API', () => { duplicateBarcodes: unknown[] locationPathInconsistencies: unknown[] containersWithNoGridPosition: unknown[] + containersWithInvalidGridPosition: unknown[] } expect(Array.isArray(data.emptyCollections)).toBe(true) expect(Array.isArray(data.collectionsWithMissingLocation)).toBe(true) @@ -258,6 +262,7 @@ describe('Data audit API', () => { expect(Array.isArray(data.duplicateBarcodes)).toBe(true) expect(Array.isArray(data.locationPathInconsistencies)).toBe(true) expect(Array.isArray(data.containersWithNoGridPosition)).toBe(true) + expect(data.containersWithInvalidGridPosition).toEqual([]) }) it('includes containers with missing specimen in report', async () => { @@ -491,5 +496,53 @@ describe('Data audit API', () => { expect(match!.containerType).toBe('micronix_tube') expect(match!.collectionId).toBe(plate.id) }) + + it('lists Containers whose stored position fails the Grid position rule, and no valid or missing ones', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'ShelfBadPos', description: '' }) + const loc = await createTestLocation(ctx.db, { name: 'LocBadPos', storageTypeId: String(storageType.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'PlateBadPos', locationId: loc.id }) + const cvBox = await createTestCryovialBox(ctx.db, { name: 'BoxBadPos', locationId: loc.id }) + const specType = await createTestSpecimenType(ctx.db, { name: 'STBadPos' }) + const spec = await createTestSpecimen(ctx.db, specType.id) + const unit = await createTestUnit(ctx.db, { symbol: 'uL-badpos', name: 'microliter', category: 'volume' }) + const newContainer = async () => (await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id })).id + + const ids: Record = {} + for (const [barcode, position] of [ + ['MX-OK', 'H12'], + ['MX-NULL', null], + ['MX-BARCODE', 'A1145854300'], + ['MX-UNPADDED', 'A1'], + ['MX-OUTSIDE', 'J10'], + ] as const) { + ids[barcode] = await newContainer() + await createTestMicronixTube(ctx.db, { containerId: ids[barcode]!, collectionId: plate.id, barcode, position }) + } + for (const [barcode, position] of [ + ['CV-OK', 'J10'], + ['CV-DASH', '--A01'], + ['CV-NA', 'NA01'], + ] as const) { + ids[barcode] = await newContainer() + await createTestCryovialTube(ctx.db, { containerId: ids[barcode]!, collectionId: cvBox.id, barcode, position }) + } + + const res = await ctx.request('/api/admin/data-audit/integrity-report', { method: 'GET' }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { + containersWithInvalidGridPosition: Array<{ id: number; containerType: string; collectionId: number; position: string }> + } + const byId = (a: { id: number }, b: { id: number }) => a.id - b.id + expect([...data.containersWithInvalidGridPosition].sort(byId)).toEqual( + [ + { id: ids['MX-BARCODE']!, containerType: 'micronix_tube', collectionId: plate.id, position: 'A1145854300' }, + { id: ids['MX-UNPADDED']!, containerType: 'micronix_tube', collectionId: plate.id, position: 'A1' }, + { id: ids['MX-OUTSIDE']!, containerType: 'micronix_tube', collectionId: plate.id, position: 'J10' }, + { id: ids['CV-DASH']!, containerType: 'cryovial_tube', collectionId: cvBox.id, position: '--A01' }, + { id: ids['CV-NA']!, containerType: 'cryovial_tube', collectionId: cvBox.id, position: 'NA01' }, + ].sort(byId), + ) + }) }) }) diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index a2ea1f23..7a2a7567 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -29,6 +29,7 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Storage container tag orphans.** Tag links that reference a deleted container or tag. - **Duplicate barcodes (micronix).** Micronix tubes that share the same barcode (schema expects unique barcodes). Duplicate barcodes on cryovial tubes are **not** treated as a data integrity issue. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. +- **Containers with an invalid Grid position.** Tubes or wells whose stored position is not a well or slot of their plate (A01–H12) or box (A01–J10), or is not stored as a row letter plus a two-digit column (`A1` instead of `A01`). Such a tube shows as unplaced, and another tube can be put in its cell. These count toward the overview issue total. Move each one to its real position, or clear the position. An upgrade to schema version 10 repairs the legacy cases automatically: it strips leading dashes (`--A01` becomes `A01`) and clears positions that are the tube's own barcode or `NA`. - **Containers with no grid position.** Tubes or wells in a collection with no well or slot. Legacy rows are allowed and do **not** count toward the overview issue total. Assign a position via edit or scan move when the well is known. ## Empty collections diff --git a/packages/web/src/lib/api/admin.ts b/packages/web/src/lib/api/admin.ts index 19b57db1..12c554bf 100644 --- a/packages/web/src/lib/api/admin.ts +++ b/packages/web/src/lib/api/admin.ts @@ -87,6 +87,14 @@ export interface ContainerWithNoGridPositionItem { collectionId: number } +export interface ContainerWithInvalidGridPositionItem { + id: number + containerType: 'micronix_tube' | 'cryovial_tube' | 'static_well' + collectionId: number + /** The stored position, as stored. */ + position: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -100,6 +108,7 @@ export interface IntegrityReport { duplicateBarcodes: DuplicateBarcodeItem[] locationPathInconsistencies: LocationPathInconsistencyItem[] containersWithNoGridPosition: ContainerWithNoGridPositionItem[] + containersWithInvalidGridPosition: ContainerWithInvalidGridPositionItem[] } export interface CollectionWithMissingLocationItem { diff --git a/packages/web/src/pages/AdminDataIntegrityOverview.tsx b/packages/web/src/pages/AdminDataIntegrityOverview.tsx index 5d1b2d9b..dfe007f7 100644 --- a/packages/web/src/pages/AdminDataIntegrityOverview.tsx +++ b/packages/web/src/pages/AdminDataIntegrityOverview.tsx @@ -39,7 +39,8 @@ export default function AdminDataIntegrityOverview() { report.derivationBrokenRefs.length + report.storageContainerTagOrphans.length + report.duplicateBarcodes.length + - report.locationPathInconsistencies.length + report.locationPathInconsistencies.length + + report.containersWithInvalidGridPosition.length // containersWithNoGridPosition is informational and is not an integrity issue count return ( @@ -71,7 +72,7 @@ export default function AdminDataIntegrityOverview() {

{integrityIssueCount}

- Orphans, broken references, duplicate micronix barcodes, path inconsistencies. + Orphans, broken references, duplicate micronix barcodes, path inconsistencies, invalid Grid positions.

diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index d7e44eb2..31d0560a 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -386,6 +386,46 @@ export default function AdminDataIntegrityReport() { + +
+ + + + + + + + + + + {report.containersWithInvalidGridPosition.map((row) => ( + + + + + + + ))} + +
Container IDTypeCollection IDStored position
+ + {row.id} + + {row.containerType} + + {row.collectionId} + + {row.position}
+
+ + { }) }) + it('counts Containers with an invalid Grid position as integrity issues', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + containersWithInvalidGridPosition: [ + { id: 1, containerType: 'cryovial_tube', collectionId: 790, position: '--A01' }, + { id: 2, containerType: 'micronix_tube', collectionId: 254, position: 'A1145854300' }, + ], + }) + await render() + await waitFor(() => { + expect(screen.getByRole('link', { name: /integrity report/i })).toHaveTextContent('2') + }) + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() diff --git a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx index c2baf923..428a37d6 100644 --- a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx +++ b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx @@ -17,6 +17,7 @@ function emptyIntegrityReport() { duplicateBarcodes: [], locationPathInconsistencies: [], containersWithNoGridPosition: [], + containersWithInvalidGridPosition: [], } } @@ -56,6 +57,20 @@ describe('AdminDataIntegrityReport', () => { }) }) + it('shows Containers with an invalid Grid position, with the stored position and a Collection link', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + containersWithInvalidGridPosition: [ + { id: 51, containerType: 'cryovial_tube', collectionId: 790, position: '--A01' }, + ], + }) + await render() + expect(await screen.findByText('Containers with an invalid Grid position')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '51' })).toHaveAttribute('href', '/containers/51') + expect(screen.getByRole('link', { name: '790' })).toHaveAttribute('href', '/collections/cryovial-boxes/790') + expect(screen.getByText('--A01')).toBeInTheDocument() + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() From 66a5e9de44593702fa6626c6e4236622faf158a1 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:48:45 -0700 Subject: [PATCH 18/54] fix(containers): keep Container quantities within bounds on every write (#109) One rule in @sampledb/contract (container-quantity.ts) decides the bounds: total and remaining are 0 or more, remaining is no more than total, and a quantity used is greater than 0 and no more than the Parent container's remaining quantity. Every write path uses it: - PATCH /containers/:id rejects a negative remaining quantity, or one above the Container's total, with a 400 before any write. - Derivation checks the Child container quantity and the quantity used up front and throws a 400 ValidationError. It no longer clamps an overdraw to 0, and a negative quantity used can no longer refill the parent. - The derivations CSV validate step reports these as row errors and adds up rows that draw from the same Parent container. Import rejects with the row number. The unused validateParentQuantity setting is removed. - Specimen registration (bulk combined import, POST /specimens, control batches, Child containers) checks the resolved quantities, so bulk import errors carry their row numbers. Web: the Container edit modal checks the bounds against the total before saving and shows the API's message. The derivations bulk import review step lists the invalid rows from validation. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/bulk-combined-validate.test.ts | 79 +++++++++++- .../src/lib/__tests__/derivations-csv.test.ts | 72 +++++++++++ packages/api/src/lib/derivations-csv.ts | 76 ++++++++---- packages/api/src/lib/derivations.ts | 68 ++++++----- .../api/src/lib/specimen-registration/plan.ts | 5 + .../src/routes/__tests__/containers.test.ts | 56 +++++++++ .../src/routes/__tests__/derivations.test.ts | 114 ++++++++++++++++++ packages/api/src/routes/containers.ts | 11 +- packages/api/src/routes/imports.ts | 1 - .../src/__tests__/container-quantity.test.ts | 66 ++++++++++ packages/contract/src/container-quantity.ts | 48 ++++++++ packages/contract/src/index.ts | 6 + .../docs/guides/features/derivations.md | 2 +- .../web/src/components/ContainerEditModal.tsx | 22 +++- .../__tests__/ContainerEditModal.test.tsx | 70 +++++++++++ packages/web/src/lib/api/derivations.ts | 1 - packages/web/src/pages/ContainerDetail.tsx | 1 + .../web/src/pages/DerivationsBulkImport.tsx | 16 ++- .../__tests__/DerivationsBulkImport.test.tsx | 26 ++++ 19 files changed, 673 insertions(+), 67 deletions(-) create mode 100644 packages/contract/src/__tests__/container-quantity.test.ts create mode 100644 packages/contract/src/container-quantity.ts diff --git a/packages/api/src/lib/__tests__/bulk-combined-validate.test.ts b/packages/api/src/lib/__tests__/bulk-combined-validate.test.ts index 793e8a30..d95def06 100644 --- a/packages/api/src/lib/__tests__/bulk-combined-validate.test.ts +++ b/packages/api/src/lib/__tests__/bulk-combined-validate.test.ts @@ -9,8 +9,9 @@ import { createTestUnit, } from '../../__tests__/helpers/factories' import { setContainerDefaults } from '../settings' -import { validateBulkCombinedPayload } from '../bulk-combined-import' -import { specimenTypeContainerType, containerTypeUnit } from '../../db/schema' +import { runBulkCombinedImport, validateBulkCombinedPayload } from '../bulk-combined-import' +import { ValidationError } from '../error-handler' +import { specimenTypeContainerType, containerTypeUnit, storageContainer } from '../../db/schema' import type { Database } from '../../db/client' import { utcNow } from '../datetime' describe('bulk-combined-validate', () => { @@ -349,4 +350,78 @@ describe('bulk-combined-validate', () => { expect(result.errors[0]!.message.toLowerCase()).toContain('study') }) }) + + describe('container quantity bounds', () => { + async function seedMicronixSetup() { + const study = await createTestStudy(testDb, { title: 'Study 1', shortCode: 'ST1' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'DNA' }) + const now = utcNow() + await testDb.insert(specimenTypeContainerType).values({ specimenTypeId: specimenType.id, containerType: 'micronix_tube', created: now }) + const defaults = { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' } + await setContainerDefaults(testDb, { micronix_tube: defaults, cryovial_tube: defaults, paper: defaults, static_well: defaults }) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await testDb.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(testDb, { name: 'PL1', locationId: location.id }) + return { study, plate } + } + + function specimenRow(plateId: number, row: number, quantities: { totalQuantity?: number; remainingQuantity?: number }) { + return { + specimenTypeName: 'DNA', + collectionDate: '2024-01-15', + rowIndex: row, + container: { + containerType: 'micronix_tube' as const, + barcode: `MT-${row}`, + collection: { type: 'micronix_plate' as const, id: plateId, position: `A0${row}` }, + ...quantities, + }, + } + } + + it('rejects negative quantities and remaining greater than total, with row numbers', async () => { + const { study, plate } = await seedMicronixSetup() + + const result = await validateBulkCombinedPayload(testDb, { + atomicMode: 'full_file', + studyShortCode: study.shortCode, + subjects: [ + { + subjectName: 'Subj1', + specimens: [ + specimenRow(plate.id, 1, { totalQuantity: 5, remainingQuantity: 5 }), + specimenRow(plate.id, 2, { totalQuantity: -1 }), + specimenRow(plate.id, 3, { totalQuantity: 5, remainingQuantity: -2 }), + specimenRow(plate.id, 4, { totalQuantity: 5, remainingQuantity: 6 }), + ], + }, + ], + }) + + expect(result.valid).toBe(false) + expect(result.errors.map((e) => [e.rowIndex, e.message])).toEqual([ + [2, expect.stringContaining('Total quantity must be 0 or more (got -1).')], + [3, expect.stringContaining('Remaining quantity must be 0 or more (got -2).')], + [4, expect.stringContaining('Remaining quantity (6) cannot exceed total quantity (5).')], + ]) + }) + + it('rejects the import with the row number and creates nothing', async () => { + const { study, plate } = await seedMicronixSetup() + + const error = await runBulkCombinedImport(testDb, { + atomicMode: 'full_file', + studyShortCode: study.shortCode, + subjects: [{ subjectName: 'Subj1', specimens: [specimenRow(plate.id, 7, { totalQuantity: 2, remainingQuantity: 3 })] }], + }).catch((e: unknown) => e) + + expect(error).toBeInstanceOf(ValidationError) + expect((error as ValidationError).details.errors).toEqual([ + expect.objectContaining({ rowIndex: 7, message: expect.stringContaining('Remaining quantity (3) cannot exceed total quantity (2).') }), + ]) + expect(await testDb.select().from(storageContainer)).toHaveLength(0) + }) + }) }) diff --git a/packages/api/src/lib/__tests__/derivations-csv.test.ts b/packages/api/src/lib/__tests__/derivations-csv.test.ts index 5aef2c8a..abe94945 100644 --- a/packages/api/src/lib/__tests__/derivations-csv.test.ts +++ b/packages/api/src/lib/__tests__/derivations-csv.test.ts @@ -399,6 +399,78 @@ describe('derivations-csv', () => { expect(after?.remainingQuantity).toBe(10) }) + it('rejects a quantity used of 0 or less, with the row', async () => { + const { parent } = await seedParent() + const csv = [`${header},quantity_used`, `${parent.id},PlateDef,B01,C1,0`, `${parent.id},PlateDef,B02,C2,-100`].join('\n') + + const validation = await validateDerivationsCsv(testDb, csv, settings) + + expect(validation.rows.map((r) => r.error)).toEqual([ + 'Quantity used must be greater than 0 (got 0).', + 'Quantity used must be greater than 0 (got -100).', + ]) + + const result = await importDerivationsFromCsv(testDb, csv, { settings }) + + expect(result.rows.find((r) => !r.success && r.index === 0)?.error).toBe( + 'Row 1: Quantity used must be greater than 0 (got 0). No derivations were created; please fix the error and try again.', + ) + const after = await testDb.select().from(storageContainer).where(eq(storageContainer.id, parent.id)).get() + expect(after?.remainingQuantity).toBe(10) + }) + + it('rejects a shared quantity used of 0 or less', async () => { + const { parent } = await seedParent() + const csv = [header, `${parent.id},PlateDef,B01,C1`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, { ...settings, quantityUsed: -1 }) + + expect(result.rows[0].error).toBe('Quantity used must be greater than 0 (got -1).') + }) + + it('rejects a negative Child container quantity', async () => { + const { parent } = await seedParent() + const csv = [`${header},quantity`, `${parent.id},PlateDef,B01,C1,-2`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, settings) + + expect(result.rows[0].error).toBe('Child container: Total quantity must be 0 or more (got -2).') + }) + + it('rejects rows that together draw more than the Parent container holds', async () => { + const { parent } = await seedParent() + const csv = [ + `${header},quantity_used`, + `${parent.id},PlateDef,B01,C1,6`, + `${parent.id},PlateDef,B02,C2,4`, + `${parent.id},PlateDef,B03,C3,1`, + ].join('\n') + + const validation = await validateDerivationsCsv(testDb, csv, settings) + + expect(validation.rows.map((r) => r.valid)).toEqual([true, true, false]) + expect(validation.rows[2].error).toBe( + "Quantity used (1) exceeds the Parent container's remaining quantity (0). Earlier rows in this file draw from the same Parent container.", + ) + + const result = await importDerivationsFromCsv(testDb, csv, { settings }) + + expect(result.rows.find((r) => r.index === 2)?.error).toBe( + "Row 3: Quantity used (1) exceeds the Parent container's remaining quantity (0). No derivations were created; please fix the error and try again.", + ) + const after = await testDb.select().from(storageContainer).where(eq(storageContainer.id, parent.id)).get() + expect(after?.remainingQuantity).toBe(10) + }) + + it('rejects a single row drawing more than the Parent container holds', async () => { + const { parent } = await seedParent() + const csv = [`${header},quantity_used`, `${parent.id},PlateDef,B01,C1,11`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, settings) + + expect(result.rows[0].error).toBe("Quantity used (11) exceeds the Parent container's remaining quantity (10).") + }) + it('rejects paper rows whose box does not exist', async () => { const { parent } = await seedParent() const csv = ['parent_container_id,container_type,box_name,sheet_name,sublabel', `${parent.id},paper,No Such Box,Sheet-1,S1`].join('\n') diff --git a/packages/api/src/lib/derivations-csv.ts b/packages/api/src/lib/derivations-csv.ts index 119a86e1..794bc66e 100644 --- a/packages/api/src/lib/derivations-csv.ts +++ b/packages/api/src/lib/derivations-csv.ts @@ -14,8 +14,11 @@ import { } from '../db/schema' import { and, eq, sql } from 'drizzle-orm' import { + containerQuantityBoundsError, csvRowToContainerWriteInput, parseCsv as parseCsvCells, + quantityDrawError, + quantityUsedError, PAPER_DERIVATION_USE_SUBLABEL_NOT_CONTAINER_BARCODE, PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type ContainerCsvRow, @@ -69,7 +72,6 @@ export interface BulkDerivationSettings { // Validation flags validateSourceSpecimenType?: boolean - validateParentQuantity?: boolean } export interface DerivationCsvRow { @@ -176,6 +178,27 @@ function rowValueError(row: DerivationCsvRow): string | null { return null } +/** The quantity used a row draws from its Parent container, or undefined when it draws none. */ +function rowQuantityDraw(row: DerivationCsvRow, settings: BulkDerivationSettings | undefined): number | undefined { + const reduce = parseBoolean(row.reduce_parent_quantity) ?? settings?.reduceParentQuantity ?? true + return reduce ? rowQuantityUsed(row, settings) : undefined +} + +function rowQuantityUsed(row: DerivationCsvRow, settings: BulkDerivationSettings | undefined): number | undefined { + return row.quantity_used ? parseNumber(row.quantity_used) : settings?.quantityUsed +} + +/** A row's Child container quantity or quantity used out of bounds, before looking at the Parent container. */ +function rowQuantityError(row: DerivationCsvRow, settings: BulkDerivationSettings | undefined): string | null { + const quantity = row.quantity ? parseNumber(row.quantity) : settings?.quantity + if (quantity != null) { + const childError = containerQuantityBoundsError({ totalQuantity: quantity, remainingQuantity: quantity }) + if (childError) return `Child container: ${childError}` + } + const quantityUsed = rowQuantityUsed(row, settings) + return quantityUsed == null ? null : quantityUsedError(quantityUsed) +} + function parseNumber(value?: string): number | undefined { if (value == null || value.trim() === '') return undefined const n = Number(value) @@ -385,6 +408,8 @@ export async function validateDerivationsCsv( let invalidCount = 0 let warningCount = 0 let firstParentSpecimenTypeId: number | null = null + /** Parent container remaining quantity after the draws of earlier rows. */ + const parentRemaining = new Map() for (let i = 0; i < rows.length; i++) { const row = rows[i] @@ -490,28 +515,35 @@ export async function validateDerivationsCsv( } } - // Validate parent quantity if enabled - if (settings?.validateParentQuantity && parentContainerId) { - const parentContainer = await database - .select({ - remainingQuantity: storageContainer.remainingQuantity, - unitId: storageContainer.unitId, - }) - .from(storageContainer) - .where(eq(storageContainer.id, parentContainerId)) - .get() - - if (parentContainer) { - const quantityUsed = row.quantity_used - ? parseNumber(row.quantity_used) - : settings.quantityUsed - - if (quantityUsed && parentContainer.remainingQuantity !== null && parentContainer.remainingQuantity < quantityUsed) { - validationRow.warnings = validationRow.warnings || [] - validationRow.warnings.push(`Insufficient parent quantity: ${parentContainer.remainingQuantity} available, ${quantityUsed} requested`) - warningCount++ - } + // Same bounds createDerivation enforces; rows drawing from one Parent container add up. + const quantityError = rowQuantityError(row, settings) + if (quantityError) { + validationRow.error = quantityError + validationRows.push(validationRow) + invalidCount++ + continue + } + const draw = rowQuantityDraw(row, settings) + if (draw != null && parentContainerId) { + let remaining = parentRemaining.get(parentContainerId) + if (remaining === undefined) { + const parentContainer = await database + .select({ remainingQuantity: storageContainer.remainingQuantity }) + .from(storageContainer) + .where(eq(storageContainer.id, parentContainerId)) + .get() + remaining = parentContainer?.remainingQuantity ?? null + } + const drawError = quantityDrawError(draw, remaining) + if (drawError) { + validationRow.error = parentRemaining.has(parentContainerId) + ? `${drawError} Earlier rows in this file draw from the same Parent container.` + : drawError + validationRows.push(validationRow) + invalidCount++ + continue } + parentRemaining.set(parentContainerId, remaining == null ? null : remaining - draw) } // Resolve collection and track status diff --git a/packages/api/src/lib/derivations.ts b/packages/api/src/lib/derivations.ts index 7855e70f..365c61cc 100644 --- a/packages/api/src/lib/derivations.ts +++ b/packages/api/src/lib/derivations.ts @@ -1,4 +1,4 @@ -import type { ContainerWriteInput } from '@sampledb/contract' +import { containerQuantityBoundsError, quantityDrawError, quantityUsedError, type ContainerWriteInput } from '@sampledb/contract' import type { Database } from '../db/client' import type { ExtractTablesWithRelations } from 'drizzle-orm' import type { SQLiteTransaction } from 'drizzle-orm/sqlite-core' @@ -17,6 +17,7 @@ import { utcNow } from './datetime' import { createContainerForSpecimen, pickContainerQuantity } from './container-creation' import { withWriteTransaction } from '../db/write-transaction' import { insertedRow } from './inserted-row' +import { ValidationError } from './error-handler' export type DerivationType = string @@ -37,7 +38,8 @@ export interface CreateDerivationInput { } export interface DerivationWarning { - code: 'INSUFFICIENT_PARENT_QUANTITY' | 'PARENT_QUANTITY_ZERO_OR_NEGATIVE' | 'INCOMPATIBLE_UNITS' + // Never produced: units are not converted or compared yet. Quantity bounds are errors (derivationQuantityError). + code: 'INCOMPATIBLE_UNITS' message: string } @@ -147,46 +149,45 @@ async function resolveUnitIdForChild( return getDefaultUnit(database as Database, containerType) } -async function adjustParentQuantity( - database: DatabaseOrTransaction, +/** + * Why this Derivation's quantities are out of bounds, or null. The Child container's quantity + * starts as both its total and remaining; a quantity used must be greater than 0 and, when it + * reduces the Parent container, no more than the Parent container's remaining quantity. + */ +function derivationQuantityError( parent: typeof storageContainer.$inferSelect, - quantityUsed?: number, - reduceParentQuantity?: boolean, -): Promise<{ updatedParent: typeof storageContainer.$inferSelect; warnings: DerivationWarning[] }> { - const warnings: DerivationWarning[] = [] - - if (!reduceParentQuantity || quantityUsed == null) { - return { updatedParent: parent, warnings } - } - - if (parent.remainingQuantity == null) { - return { updatedParent: parent, warnings } + input: CreateDerivationInput, +): string | null { + if (input.quantity != null) { + const childError = containerQuantityBoundsError({ totalQuantity: input.quantity, remainingQuantity: input.quantity }) + if (childError) return `Child container: ${childError}` } + if (input.quantityUsed == null) return null + return (input.reduceParentQuantity ?? true) + ? quantityDrawError(input.quantityUsed, parent.remainingQuantity) + : quantityUsedError(input.quantityUsed) +} - if (parent.remainingQuantity <= 0) { - warnings.push({ - code: 'PARENT_QUANTITY_ZERO_OR_NEGATIVE', - message: 'Parent container remaining quantity is zero or negative; quantity reduction will be clamped.', - }) - } else if (parent.remainingQuantity < quantityUsed) { - warnings.push({ - code: 'INSUFFICIENT_PARENT_QUANTITY', - message: 'Parent container remaining quantity is less than quantity used; quantity reduction will proceed and be clamped at zero.', - }) +async function reduceParentQuantity( + database: DatabaseOrTransaction, + parent: typeof storageContainer.$inferSelect, + quantityUsed: number | undefined, + reduce: boolean, +): Promise { + if (!reduce || quantityUsed == null || parent.remainingQuantity == null) { + return parent } - const newRemaining = Math.max(0, parent.remainingQuantity - quantityUsed) - const [updated] = await database .update(storageContainer) .set({ - remainingQuantity: newRemaining, + remainingQuantity: parent.remainingQuantity - quantityUsed, lastUpdated: utcNow(), }) .where(eq(storageContainer.id, parent.id)) .returning() - return { updatedParent: updated, warnings } + return updated } export async function createDerivation( @@ -213,6 +214,11 @@ async function createDerivationInTx( throw new Error('Parent container not found') } + const quantityError = derivationQuantityError(parent, input) + if (quantityError) { + throw new ValidationError(quantityError) + } + const derivedSpecimenId = await findOrCreateDerivedSpecimen(database, parent.specimenId, input.specimenTypeName) const specType = await database @@ -256,7 +262,7 @@ async function createDerivationInTx( throw new Error('Child container not found after creation') } - const { updatedParent, warnings } = await adjustParentQuantity( + const updatedParent = await reduceParentQuantity( database, parent, input.quantityUsed, @@ -290,6 +296,6 @@ async function createDerivationInTx( parentContainer: updatedParent, childContainer: child, specimen: derivedSpec, - warnings, + warnings: [], } } diff --git a/packages/api/src/lib/specimen-registration/plan.ts b/packages/api/src/lib/specimen-registration/plan.ts index 5f9e07e2..31300ddb 100644 --- a/packages/api/src/lib/specimen-registration/plan.ts +++ b/packages/api/src/lib/specimen-registration/plan.ts @@ -3,6 +3,7 @@ * every problem. Reads only; the write step trusts the plan. */ import { and, eq, isNull } from 'drizzle-orm' +import { containerQuantityBoundsError } from '@sampledb/contract' import type { Database } from '../../db/client' import { controlBatch, @@ -246,6 +247,10 @@ async function planContainer( totalQuantity = input.totalQuantity ?? (await getDefaultTotalQuantity(database, containerType)) remainingQuantity = input.remainingQuantity ?? input.totalQuantity ?? (await getDefaultRemainingQuantity(database, containerType)) + const quantityError = containerQuantityBoundsError({ totalQuantity, remainingQuantity }) + if (quantityError) { + problems.push({ field: totalQuantity < 0 ? 'totalQuantity' : 'remainingQuantity', message: quantityError }) + } } catch (error) { problems.push({ field: 'unitId', message: error instanceof Error ? error.message : 'Container defaults are not configured' }) } diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index cd9aa39d..54b751d5 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -589,4 +589,60 @@ describe('Containers API', () => { expect(logsAfter.at(-1)?.message).toBe('enrichment failed') }) }) + + describe('PATCH /api/containers/:id keeps remaining quantity within bounds', () => { + /** A Container with total 10 and remaining 4. */ + async function seedQuantityContainer() { + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Whole Blood' }) + const specimenId = (await createTestSpecimen(ctx.db, specimenType.id)).id + const container = await createTestStorageContainer(ctx.db, { specimenId }) + await ctx.db + .update(storageContainer) + .set({ totalQuantity: 10, remainingQuantity: 4, comment: 'original' }) + .where(eq(storageContainer.id, container.id)) + return container.id + } + + async function remainingOf(id: number) { + return (await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, id)).get())?.remainingQuantity + } + + it('returns 400 for a negative remaining quantity and leaves the Container unchanged', async () => { + const id = await seedQuantityContainer() + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { comment: 'changed', remainingQuantity: -5 } }) + + expect(res.status).toBe(400) + const body = (await res.json()) as { error: string; errorCode: string } + expect(body.errorCode).toBe('VALIDATION_ERROR') + expect(body.error).toBe('Remaining quantity must be 0 or more (got -5).') + expect(await remainingOf(id)).toBe(4) + const row = await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, id)).get() + expect(row?.comment).toBe('original') + }) + + it('returns 400 for a remaining quantity greater than total', async () => { + const id = await seedQuantityContainer() + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { remainingQuantity: 1e9 } }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe( + 'Remaining quantity (1000000000) cannot exceed total quantity (10).', + ) + expect(await remainingOf(id)).toBe(4) + }) + + it('saves a remaining quantity of 0 and one equal to total', async () => { + const id = await seedQuantityContainer() + + const zero = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { remainingQuantity: 0 } }) + expect(zero.status).toBe(200) + expect(await remainingOf(id)).toBe(0) + + const full = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { remainingQuantity: 10 } }) + expect(full.status).toBe(200) + expect(await remainingOf(id)).toBe(10) + }) + }) }) diff --git a/packages/api/src/routes/__tests__/derivations.test.ts b/packages/api/src/routes/__tests__/derivations.test.ts index 47502ec2..30c8e0eb 100644 --- a/packages/api/src/routes/__tests__/derivations.test.ts +++ b/packages/api/src/routes/__tests__/derivations.test.ts @@ -5,6 +5,18 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createDerivationsRoutes } from '../derivations' +import { + createTestLocation, + createTestMicronixPlate, + createTestSpecimen, + createTestSpecimenType, + createTestStorageType, + createTestUnit, +} from '../../__tests__/helpers/factories' +import { setContainerDefaults } from '../../lib/settings' +import { containerDerivation, containerTypeUnit, micronixTube, specimenTypeContainerType, storageContainer } from '../../db/schema' +import { utcNow } from '../../lib/datetime' +import { eq } from 'drizzle-orm' const BASE = '/api/derivations' @@ -98,6 +110,108 @@ describe('Derivations API', () => { }) }) + describe('POST /api/containers/:id/derive quantity bounds', () => { + let parentId: number + let targetPlateId: number + + /** A micronix Parent container with total 10 and remaining 4, and an empty target plate. */ + beforeEach(async () => { + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const defaults = { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' } + await setContainerDefaults(ctx.db, { + micronix_tube: defaults, + cryovial_tube: defaults, + paper: defaults, + static_well: defaults, + }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'DNA' }) + const now = utcNow() + await ctx.db.insert(specimenTypeContainerType).values({ specimenTypeId: specimenType.id, containerType: 'micronix_tube', created: now }) + await ctx.db.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const parentSpecimen = await createTestSpecimen(ctx.db, specimenType.id) + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const location = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const sourcePlate = await createTestMicronixPlate(ctx.db, { name: 'SourcePlate', locationId: location.id }) + targetPlateId = (await createTestMicronixPlate(ctx.db, { name: 'TargetPlate', locationId: location.id })).id + const [parent] = await ctx.db + .insert(storageContainer) + .values({ specimenId: parentSpecimen.id, unitId: unit.id, totalQuantity: 10, remainingQuantity: 4, created: now, lastUpdated: now }) + .returning() + parentId = parent!.id + await ctx.db.insert(micronixTube).values({ id: parentId, collectionId: sourcePlate.id, barcode: 'MT-PARENT', position: 'A01' }) + }) + + function derive(fields: Record) { + return ctx.request(`${BASE}/containers/${parentId}/derive`, { + method: 'POST', + json: { + derivationType: 'aliquot', + specimenTypeName: 'DNA', + container: { + containerType: 'micronix_tube', + barcode: 'MT-CHILD', + collection: { type: 'micronix_plate', id: targetPlateId, position: 'A01' }, + }, + ...fields, + }, + }) + } + + async function parentRemaining() { + return (await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, parentId)).get())?.remainingQuantity + } + + async function expectNothingDerived() { + expect(await parentRemaining()).toBe(4) + expect(await ctx.db.select().from(containerDerivation)).toHaveLength(0) + expect(await ctx.db.select().from(storageContainer)).toHaveLength(1) + } + + it.each([ + [0, 'Quantity used must be greater than 0 (got 0).'], + [-100, 'Quantity used must be greater than 0 (got -100).'], + ])('returns 400 for a quantity used of %d and changes nothing', async (quantityUsed, message) => { + const res = await derive({ quantityUsed }) + + expect(res.status).toBe(400) + const body = (await res.json()) as { error: string; errorCode: string } + expect(body).toMatchObject({ error: message, errorCode: 'VALIDATION_ERROR' }) + await expectNothingDerived() + }) + + it('returns 400 for a quantity used greater than the Parent container remaining quantity', async () => { + const res = await derive({ quantityUsed: 5 }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe( + "Quantity used (5) exceeds the Parent container's remaining quantity (4).", + ) + await expectNothingDerived() + }) + + it('returns 400 for a negative Child container quantity', async () => { + const res = await derive({ quantity: -1, quantityUsed: 1 }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe('Child container: Total quantity must be 0 or more (got -1).') + await expectNothingDerived() + }) + + it('draws all that remains', async () => { + const res = await derive({ quantityUsed: 4 }) + + expect(res.status).toBe(200) + expect(await parentRemaining()).toBe(0) + }) + + it('does not bound a quantity used that leaves the Parent container unreduced', async () => { + const res = await derive({ quantityUsed: 100, reduceParentQuantity: false }) + + expect(res.status).toBe(200) + expect(await parentRemaining()).toBe(4) + }) + }) + describe('GET /api/containers/:id/source', () => { it('returns 400 for invalid container ID', async () => { const res = await ctx.request(`${BASE}/containers/notanid/source`, { diff --git a/packages/api/src/routes/containers.ts b/packages/api/src/routes/containers.ts index eaeeb44f..cb325ad2 100644 --- a/packages/api/src/routes/containers.ts +++ b/packages/api/src/routes/containers.ts @@ -3,6 +3,7 @@ import type { Database } from '../db/client' import { storageContainer, micronixTube, cryovialTube, paper, staticWell, tag, storageContainerTag } from '../db/schema' import { eq, and, sql, inArray } from 'drizzle-orm' import { z } from 'zod' +import { containerQuantityBoundsError } from '@sampledb/contract' import { validatePage, validateLimit } from '../lib/constants' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { utcNow } from '../lib/datetime' @@ -217,13 +218,21 @@ containers.patch('/:id', memberMiddleware, async (c) => { const userId = c.get('user')?.id const updated = await withWriteTransaction(database, async (tx) => { - const container = await tx.select({ id: storageContainer.id }).from(storageContainer).where(eq(storageContainer.id, id)).get() + const container = await tx + .select({ id: storageContainer.id, totalQuantity: storageContainer.totalQuantity }) + .from(storageContainer) + .where(eq(storageContainer.id, id)) + .get() if (!container) { throw new NotFoundError('Container', id) } const containerType = await containerSubtypeOf(tx, id) // Check everything before the first write. + if (remainingQuantity !== undefined) { + const quantityError = containerQuantityBoundsError({ totalQuantity: container.totalQuantity, remainingQuantity }) + if (quantityError) throw new ValidationError(quantityError) + } if (unitId !== undefined && containerType) { const validation = await validateUnitForContainerType(tx, containerType, unitId) if (!validation.valid) { diff --git a/packages/api/src/routes/imports.ts b/packages/api/src/routes/imports.ts index 573ce14d..065b2ca8 100644 --- a/packages/api/src/routes/imports.ts +++ b/packages/api/src/routes/imports.ts @@ -31,7 +31,6 @@ const bulkDerivationSettingsSchema = z.object({ quantityUsed: z.number().optional(), reduceParentQuantity: z.boolean().optional(), validateSourceSpecimenType: z.boolean().optional(), - validateParentQuantity: z.boolean().optional(), }) // Bulk import container derivations from CSV diff --git a/packages/contract/src/__tests__/container-quantity.test.ts b/packages/contract/src/__tests__/container-quantity.test.ts new file mode 100644 index 00000000..dd018941 --- /dev/null +++ b/packages/contract/src/__tests__/container-quantity.test.ts @@ -0,0 +1,66 @@ +import { describe, it, expect } from 'bun:test' +import { containerQuantityBoundsError, quantityDrawError, quantityUsedError } from '../container-quantity' + +describe('containerQuantityBoundsError', () => { + it('accepts remaining between 0 and total', () => { + expect(containerQuantityBoundsError({ totalQuantity: 10, remainingQuantity: 0 })).toBeNull() + expect(containerQuantityBoundsError({ totalQuantity: 10, remainingQuantity: 10 })).toBeNull() + expect(containerQuantityBoundsError({ totalQuantity: 0, remainingQuantity: 0 })).toBeNull() + }) + + it('accepts unset quantities', () => { + expect(containerQuantityBoundsError({})).toBeNull() + expect(containerQuantityBoundsError({ totalQuantity: null, remainingQuantity: 5 })).toBeNull() + }) + + it('rejects a negative total', () => { + expect(containerQuantityBoundsError({ totalQuantity: -1 })).toBe('Total quantity must be 0 or more (got -1).') + }) + + it('rejects a negative remaining', () => { + expect(containerQuantityBoundsError({ totalQuantity: 10, remainingQuantity: -5 })).toBe( + 'Remaining quantity must be 0 or more (got -5).', + ) + }) + + it('rejects remaining greater than total', () => { + expect(containerQuantityBoundsError({ totalQuantity: 10, remainingQuantity: 11 })).toBe( + 'Remaining quantity (11) cannot exceed total quantity (10).', + ) + }) + + it('rejects NaN', () => { + expect(containerQuantityBoundsError({ remainingQuantity: Number.NaN })).not.toBeNull() + }) +}) + +describe('quantityUsedError', () => { + it('rejects 0 and negatives', () => { + expect(quantityUsedError(0)).toBe('Quantity used must be greater than 0 (got 0).') + expect(quantityUsedError(-100)).toBe('Quantity used must be greater than 0 (got -100).') + }) + + it('accepts a positive quantity', () => { + expect(quantityUsedError(0.5)).toBeNull() + }) +}) + +describe('quantityDrawError', () => { + it('accepts drawing up to the remaining quantity', () => { + expect(quantityDrawError(5, 5)).toBeNull() + expect(quantityDrawError(1, 5)).toBeNull() + }) + + it('rejects drawing more than remains', () => { + expect(quantityDrawError(6, 5)).toBe("Quantity used (6) exceeds the Parent container's remaining quantity (5).") + expect(quantityDrawError(1, 0)).toBe("Quantity used (1) exceeds the Parent container's remaining quantity (0).") + }) + + it('rejects a quantity used of 0 or less before comparing', () => { + expect(quantityDrawError(-100, 5)).toBe('Quantity used must be greater than 0 (got -100).') + }) + + it('does not bound a Parent container with no remaining quantity', () => { + expect(quantityDrawError(5, null)).toBeNull() + }) +}) diff --git a/packages/contract/src/container-quantity.ts b/packages/contract/src/container-quantity.ts new file mode 100644 index 00000000..1d280665 --- /dev/null +++ b/packages/contract/src/container-quantity.ts @@ -0,0 +1,48 @@ +/** + * Bounds on a Container's quantities, shared by every write path: Container edit, Specimen + * registration (bulk import, POST /specimens, control batches, a Derivation's Child container), + * and Derivation (single and CSV) drawing from a Parent container. + * + * A Container's remaining quantity stays between 0 and its total. A Derivation draws a quantity + * used greater than 0 and no more than the Parent container's remaining quantity. + */ + +export type ContainerQuantityBounds = { + totalQuantity?: number | null + remainingQuantity?: number | null +} + +/** Why a Container's total and remaining quantities are out of bounds, or null when they are in bounds. */ +export function containerQuantityBoundsError({ totalQuantity, remainingQuantity }: ContainerQuantityBounds): string | null { + if (totalQuantity != null && !(totalQuantity >= 0)) { + return `Total quantity must be 0 or more (got ${totalQuantity}).` + } + if (remainingQuantity != null && !(remainingQuantity >= 0)) { + return `Remaining quantity must be 0 or more (got ${remainingQuantity}).` + } + if (totalQuantity != null && remainingQuantity != null && remainingQuantity > totalQuantity) { + return `Remaining quantity (${remainingQuantity}) cannot exceed total quantity (${totalQuantity}).` + } + return null +} + +/** Why a quantity used is not greater than 0, or null when it is. */ +export function quantityUsedError(quantityUsed: number): string | null { + if (!(quantityUsed > 0)) { + return `Quantity used must be greater than 0 (got ${quantityUsed}).` + } + return null +} + +/** + * Why a Derivation cannot draw `quantityUsed` from a Parent container with `remainingQuantity` + * left, or null when it can. A Parent container with no recorded remaining quantity is not bounded. + */ +export function quantityDrawError(quantityUsed: number, remainingQuantity: number | null | undefined): string | null { + const usedError = quantityUsedError(quantityUsed) + if (usedError) return usedError + if (remainingQuantity != null && quantityUsed > remainingQuantity) { + return `Quantity used (${quantityUsed}) exceeds the Parent container's remaining quantity (${remainingQuantity}).` + } + return null +} diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 1a9f103e..39660676 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -81,3 +81,9 @@ export { PAPER_DERIVATION_USE_SUBLABEL_NOT_POSITION, type PaperInboundWriteFields, } from './paper-container-inbound' +export { + containerQuantityBoundsError, + quantityDrawError, + quantityUsedError, + type ContainerQuantityBounds, +} from './container-quantity' diff --git a/packages/docs/src/content/docs/guides/features/derivations.md b/packages/docs/src/content/docs/guides/features/derivations.md index ef4185a8..750f0ea0 100644 --- a/packages/docs/src/content/docs/guides/features/derivations.md +++ b/packages/docs/src/content/docs/guides/features/derivations.md @@ -50,7 +50,7 @@ CSV columns for the derived container: For `derivation_type` in CSV, use the stored value: `aliquot` for Distribution, `dna_extraction` for DNA Extraction. The UI shows readable labels. -Optional `quantity_used` and `reduce_parent_quantity` per row, or once in **Import settings**. If you reduce quantity, the parent must have enough remaining. +Optional `quantity_used` and `reduce_parent_quantity` per row, or once in **Import settings**. `quantity_used` must be greater than 0 and `quantity` must be 0 or more. If you reduce quantity, the parent must have enough remaining for every row that draws from it, counted together; validation lists any row that would overdraw, and the import is rejected. A repository example for DBS-to-DNA from controls is `examples/derivation-control-dbs-to-dna/` (CSV and README). Operators can generate a CSV with `scripts/generate_derivation_control_dbs_to_dna_example.sh`. That is not in the app UI. diff --git a/packages/web/src/components/ContainerEditModal.tsx b/packages/web/src/components/ContainerEditModal.tsx index eae190bb..ddb06b7e 100644 --- a/packages/web/src/components/ContainerEditModal.tsx +++ b/packages/web/src/components/ContainerEditModal.tsx @@ -1,4 +1,5 @@ import { useState, useEffect, useRef } from 'react' +import { containerQuantityBoundsError } from '@sampledb/contract' import { tagsApi } from '../lib/api/reference-data'; import type { Tag } from '../lib/api/reference-data'; import { settingsApi } from '../lib/api/settings'; @@ -15,6 +16,8 @@ interface ContainerEditModalProps { id: number comment?: string | null remainingQuantity?: number + /** Upper bound for remaining quantity. */ + totalQuantity?: number tags?: Array<{ id: number; name: string }> unit?: { id: number; symbol: string } containerType?: string @@ -122,11 +125,14 @@ function ContainerEditModalForm({ // Empty string - don't update (keep current value) } else { const quantity = parseFloat(formData.remainingQuantity) - if (!isNaN(quantity) && quantity >= 0) { - updateData.remainingQuantity = quantity - } else { - throw new Error('Remaining quantity must be a valid number >= 0') + if (isNaN(quantity)) { + throw new Error('Remaining quantity must be a number') } + const boundsError = containerQuantityBoundsError({ totalQuantity: container.totalQuantity, remainingQuantity: quantity }) + if (boundsError) { + throw new Error(boundsError) + } + updateData.remainingQuantity = quantity } } @@ -228,7 +234,8 @@ function ContainerEditModalForm({ )} -
+ {/* noValidate: the quantity bounds message comes from the shared rule, not the browser. */} + {canEditBarcode(container.containerType) && (
diff --git a/packages/web/src/components/__tests__/ContainerEditModal.test.tsx b/packages/web/src/components/__tests__/ContainerEditModal.test.tsx index bf5ce7e9..388467d1 100644 --- a/packages/web/src/components/__tests__/ContainerEditModal.test.tsx +++ b/packages/web/src/components/__tests__/ContainerEditModal.test.tsx @@ -140,4 +140,74 @@ describe('ContainerEditModal', () => { await waitFor(() => expect(onSuccess).toHaveBeenCalled()) expect(containersApi.update).toHaveBeenCalledWith(8, { barcode: 'CRYO-2' }) }) + + describe('remaining quantity bounds', () => { + async function renderQuantity(onSuccess = vi.fn()) { + await render( + {}} + onSuccess={onSuccess} + container={{ id: 94, containerType: 'static_well', remainingQuantity: 4, totalQuantity: 10 }} + />, + ) + return onSuccess + } + + async function saveRemaining(value: string) { + const user = userEvent.setup() + const input = screen.getByLabelText(/Remaining Quantity/) + await user.clear(input) + await user.type(input, value) + await user.click(screen.getByRole('button', { name: 'Save Changes' })) + } + + it('shows the total as the upper bound', async () => { + await renderQuantity() + + expect(screen.getByLabelText(/Remaining Quantity/)).toHaveAttribute('max', '10') + expect(screen.getByText('Between 0 and the total quantity, 10')).toBeInTheDocument() + }) + + it('shows an error and does not save a remaining quantity greater than total', async () => { + const onSuccess = await renderQuantity() + + await saveRemaining('11') + + expect(await screen.findByText('Remaining quantity (11) cannot exceed total quantity (10).')).toBeInTheDocument() + expect(containersApi.update).not.toHaveBeenCalled() + expect(onSuccess).not.toHaveBeenCalled() + }) + + it('shows an error and does not save a negative remaining quantity', async () => { + await renderQuantity() + + await saveRemaining('-5') + + expect(await screen.findByText('Remaining quantity must be 0 or more (got -5).')).toBeInTheDocument() + expect(containersApi.update).not.toHaveBeenCalled() + }) + + it('saves a remaining quantity equal to total', async () => { + const onSuccess = await renderQuantity() + + await saveRemaining('10') + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(containersApi.update).toHaveBeenCalledWith(94, { remainingQuantity: 10 }) + }) + + it('shows the API bounds error', async () => { + vi.mocked(containersApi.update).mockRejectedValue( + Object.assign(new Error('Request failed with status code 400'), { + response: { status: 400, data: { error: 'Remaining quantity (8) cannot exceed total quantity (7).' } }, + }), + ) + await renderQuantity() + + await saveRemaining('8') + + expect(await screen.findByText('Remaining quantity (8) cannot exceed total quantity (7).')).toBeInTheDocument() + }) + }) }) diff --git a/packages/web/src/lib/api/derivations.ts b/packages/web/src/lib/api/derivations.ts index 02ee81dc..a723ea85 100644 --- a/packages/web/src/lib/api/derivations.ts +++ b/packages/web/src/lib/api/derivations.ts @@ -56,7 +56,6 @@ export interface BulkDerivationSettings { quantityUsed?: number reduceParentQuantity?: boolean validateSourceSpecimenType?: boolean - validateParentQuantity?: boolean } export interface CollectionStatus { diff --git a/packages/web/src/pages/ContainerDetail.tsx b/packages/web/src/pages/ContainerDetail.tsx index f7d89763..7c359370 100644 --- a/packages/web/src/pages/ContainerDetail.tsx +++ b/packages/web/src/pages/ContainerDetail.tsx @@ -702,6 +702,7 @@ export default function ContainerDetail() { id: container.id!, comment: container.comment, remainingQuantity: container.remainingQuantity ?? undefined, + totalQuantity: container.totalQuantity ?? undefined, tags: container.tags, unit: container.unit, containerType: effectiveContainerType, diff --git a/packages/web/src/pages/DerivationsBulkImport.tsx b/packages/web/src/pages/DerivationsBulkImport.tsx index fc5133b0..275a799f 100644 --- a/packages/web/src/pages/DerivationsBulkImport.tsx +++ b/packages/web/src/pages/DerivationsBulkImport.tsx @@ -49,7 +49,6 @@ export default function DerivationsBulkImport() { quantityUsed: undefined, reduceParentQuantity: true, validateSourceSpecimenType: false, - validateParentQuantity: false, }) const { @@ -80,6 +79,7 @@ export default function DerivationsBulkImport() { const { specimenTypes, units, allowedContainerTypes, bootstrapLoading, bootstrapError, containerTypesError } = bootstrap const formDisabled = workflowLoading || bootstrapLoading + const invalidValidationRows = validationResult?.rows.filter((row) => !row.valid && row.error) ?? [] const fileInputRef = useRef(null) const importResultsRef = useRef(null) @@ -564,7 +564,7 @@ export default function DerivationsBulkImport() { {settings.containerType !== 'paper' && (

position — Position in the collection (e.g. A01, B02).

)} -

quantity, unit_symbol, quantity_used, reduce_parent_quantity — Optional. You can set these in Import settings (same for all rows) or provide columns in the CSV (per row). Use quantity_used and reduce_parent_quantity to reduce the parent's remaining quantity (e.g. DBS spot count). Not a blocker if missing or if data is imperfect.

+

quantity, unit_symbol, quantity_used, reduce_parent_quantity — Optional. You can set these in Import settings (same for all rows) or provide columns in the CSV (per row). Use quantity_used and reduce_parent_quantity to reduce the parent's remaining quantity (e.g. DBS spot count). A row whose quantity used is 0 or less, or more than the parent has left (counting earlier rows from the same parent), is rejected.

parent_specimen_type_name is the parent specimen type (e.g. DBS, Whole Blood). The derived specimen type (e.g. DNA (DBS)) is set in Import settings or in the specimen_type_name column.

@@ -770,6 +770,18 @@ export default function DerivationsBulkImport() {

The default unit for the derived container type (e.g. µL for tubes) is used; you can change it in the review step if needed. Edit any values below before importing.

+ {invalidValidationRows.length > 0 && ( +
+

Fix these rows before importing:

+
    + {invalidValidationRows.map((row) => ( +
  • + Row {row.index + 1}: {row.error} +
    • + ))} +
+
+ )}
diff --git a/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx b/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx index 4a6ced69..e619b70c 100644 --- a/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx +++ b/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx @@ -1,7 +1,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' +import userEvent from '@testing-library/user-event' import { render, screen, waitFor } from '../../__tests__/helpers/render' import DerivationsBulkImport from '../DerivationsBulkImport' import { specimenTypesApi, unitsApi } from '../../lib/api/reference-data' +import { derivationsApi } from '../../lib/api/derivations' vi.mock('../../lib/api/reference-data', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -78,4 +80,28 @@ describe('DerivationsBulkImport', () => { expect(btn).toBeDisabled() }) }) + + it('shows a quantity bounds error on its row after validation', async () => { + const user = userEvent.setup() + const message = "Quantity used (1) exceeds the Parent container's remaining quantity (0). Earlier rows in this file draw from the same Parent container." + vi.mocked(derivationsApi.validateCsv).mockResolvedValue({ + rows: [ + { index: 0, valid: true }, + { index: 1, valid: false, error: message }, + ], + collections: [], + summary: { total: 2, valid: 1, invalid: 1, warnings: 0 }, + }) + const { container } = await render() + + const fileInput = container.querySelector('#derivation-csv-file')! + await waitFor(() => expect(fileInput).toBeEnabled()) + const csv = 'parent_container_id,plate_name,position,container_barcode,quantity_used\n1,P,A01,C1,4\n1,P,A02,C2,1\n' + await user.upload(fileInput, new File([csv], 'derivations.csv', { type: 'text/csv' })) + const validate = screen.getByRole('button', { name: /validate & continue/i }) + await waitFor(() => expect(validate).toBeEnabled()) + await user.click(validate) + + expect(await screen.findByRole('alert')).toHaveTextContent(`Row 2: ${message}`) + }) }) From f06e63f35c9f940ae5e6b531bf627e3125b4442c Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:50:45 -0700 Subject: [PATCH 19/54] docs(context): record that subject merge, qPCR experiment delete and control batch specimen delete stay member actions (#112) Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CONTEXT.md b/CONTEXT.md index 0a1399af..c871b768 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -38,6 +38,12 @@ An action that removes or rewrites recorded **Operational data** or configuratio - Deleting a non-tutorial **Study** (removes its **Subjects**, **Specimens**, and **Containers**) - Deleting **Reference data**, a **Location**, or a user account +Members may also take these actions, which cannot be undone but are part of everyday lab work, so they are not destructive actions: + +- Merging two **Subjects** +- Deleting a qPCR experiment with its plate layout and results +- Deleting one **Specimen** and its **Containers** from a **Control batch** + _Avoid_: Calling an ordinary edit (renaming a **Collection**, changing a **Container**'s quantity, removing a **Tag** from a **Container**) a destructive action. ### Provenance hierarchy From 0bfa6746e55c31efe8a79cc8d65df9292f19ff87 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:51:52 -0700 Subject: [PATCH 20/54] fix(barcodes): refuse to guess between Containers sharing a barcode (#105) Barcode lookups ended in .get() on non-unique columns, so move by barcode, Derivation parent lookup and the specimen barcode filter acted on whichever row came first. - One resolver, findContainersByBarcode in identifier-resolution.ts, returns every match. resolveContainerByBarcode throws AmbiguousBarcodeError listing the candidates; container-move's own resolver is gone. - Move by barcode fails the row with that error; the resolve endpoint returns it with the candidates, and the Scan move core shows it on the row. - Derivation parent lookup (CSV validate, import, dry run) rejects an ambiguous parent_container_barcode. - The specimen list barcode filter returns every Specimen with a matching Container. - A new or changed micronix or cryovial barcode must not match any tube's barcode, ignoring case and surrounding whitespace (registration, Derivation CSV, Container edit). An unchanged barcode is not rechecked, so Containers whose barcode is already duplicated still save. - The integrity report groups duplicate tube barcodes across types, with case and whitespace variants. Co-Authored-By: Claude Opus 5.5 --- .../src/lib/__tests__/container-move.test.ts | 46 ++++- .../src/lib/__tests__/derivations-csv.test.ts | 38 ++++ .../__tests__/identifier-resolution.test.ts | 78 +++++++- packages/api/src/lib/container-move.ts | 106 ++++------- packages/api/src/lib/data-audit.ts | 56 ++++-- packages/api/src/lib/derivations-csv.ts | 24 ++- .../parent-container-resolver.test.ts | 15 ++ .../derivations/parent-container-resolver.ts | 7 +- packages/api/src/lib/identifier-resolution.ts | 167 +++++++++++++++--- .../__tests__/specimen-registration.test.ts | 42 +++++ .../api/src/lib/specimen-registration/plan.ts | 19 +- .../specimens/__tests__/specimen-read.test.ts | 32 ++++ .../api/src/lib/specimens/specimen-read.ts | 31 ++-- .../src/routes/__tests__/collections.test.ts | 56 ++++++ .../src/routes/__tests__/containers.test.ts | 42 ++++- .../src/routes/__tests__/data-audit.test.ts | 31 ++-- packages/api/src/routes/collections.ts | 20 ++- packages/api/src/routes/containers.ts | 41 ++++- .../docs/guides/advanced/data-integrity.md | 2 +- .../bulk-operations/container-movement.md | 1 + .../docs/guides/features/derivations.md | 2 + .../guides/troubleshooting/common-issues.md | 4 +- packages/web/src/lib/api/admin.ts | 5 +- packages/web/src/lib/api/collections.ts | 10 +- .../lib/scan-move/__tests__/resolve.test.ts | 16 ++ packages/web/src/lib/scan-move/gateway.ts | 1 + packages/web/src/lib/scan-move/resolve.ts | 9 +- packages/web/src/lib/scan-move/types.ts | 2 + .../src/pages/AdminDataIntegrityReport.tsx | 10 +- .../AdminDataIntegrityOverview.test.tsx | 2 +- .../AdminDataIntegrityReport.test.tsx | 17 +- 31 files changed, 748 insertions(+), 184 deletions(-) diff --git a/packages/api/src/lib/__tests__/container-move.test.ts b/packages/api/src/lib/__tests__/container-move.test.ts index be6fff0a..850e2a0a 100644 --- a/packages/api/src/lib/__tests__/container-move.test.ts +++ b/packages/api/src/lib/__tests__/container-move.test.ts @@ -11,10 +11,13 @@ import { createTestSheet, createTestStorageContainer, createTestPaper, + createTestCryovialBox, + createTestCryovialTube, + createTestMicronixTube, } from '../../__tests__/helpers/factories' +import { resolveContainerByBarcode } from '../identifier-resolution' import { resolveContainerByPosition, - resolveContainerByBarcode, resolveContainerByIdentifier, inferCollectionTypeFromContainers, executeMoves, @@ -471,6 +474,47 @@ describe('container-move', () => { }) }) + describe('executeMoves with an ambiguous barcode', () => { + it('fails the row and lists every candidate instead of moving one of them', async () => { + const storageType = await createTestStorageType(testDb, { name: 'FreezerAmbiguous' }) + const location = await createTestLocation(testDb, { name: 'LocAmbiguous', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(testDb, { name: 'AmbPlate', locationId: location.id }) + const cryoBox = await createTestCryovialBox(testDb, { name: 'AmbBox', locationId: location.id }) + const micronix = await createTestStorageContainer(testDb, {}) + const cryovial = await createTestStorageContainer(testDb, { unitId: micronix.unitId, specimenId: micronix.specimenId }) + const other = await createTestStorageContainer(testDb, { unitId: micronix.unitId, specimenId: micronix.specimenId }) + await createTestMicronixTube(testDb, { containerId: micronix.id, collectionId: plate.id, barcode: 'SHARED', position: 'A01' }) + await createTestCryovialTube(testDb, { containerId: cryovial.id, collectionId: cryoBox.id, barcode: 'SHARED', position: 'A01' }) + await createTestMicronixTube(testDb, { containerId: other.id, collectionId: plate.id, barcode: 'UNIQUE-1', position: 'A02' }) + + const result = await executeMoves(testDb, { + atomicMode: 'best_effort', + mappings: [{ fromCollectionName: 'AmbPlate', toCollectionName: 'AmbPlate' }], + moves: [ + { identifier: { type: 'barcode', barcode: 'SHARED' }, targetPosition: 'B01' }, + { identifier: { type: 'barcode', barcode: 'UNIQUE-1' }, targetPosition: 'B02' }, + ], + }) + + expect(result).toEqual({ + success: false, + moved: 1, + errors: [ + { + row: 1, + error: + `Ambiguous barcode 'SHARED': it matches 2 containers (container ${micronix.id} (micronix_tube in AmbPlate A01), ` + + `container ${cryovial.id} (cryovial_tube in AmbBox A01)). Identify the container by its id or position instead.`, + }, + ], + }) + expect((await resolveContainerByIdentifier(testDb, { type: 'container_id', containerId: micronix.id }))?.currentPosition).toBe('A01') + expect((await resolveContainerByIdentifier(testDb, { type: 'container_id', containerId: cryovial.id }))?.currentPosition).toBe('A01') + // A container id still resolves a tube whose barcode is shared. + expect((await resolveContainerByIdentifier(testDb, { type: 'container_id', containerId: micronix.id }))?.containerId).toBe(micronix.id) + }) + }) + describe('position normalization and occupancy', () => { async function setupPlate(name = 'NormPlate') { const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) diff --git a/packages/api/src/lib/__tests__/derivations-csv.test.ts b/packages/api/src/lib/__tests__/derivations-csv.test.ts index 5aef2c8a..94ee3a0f 100644 --- a/packages/api/src/lib/__tests__/derivations-csv.test.ts +++ b/packages/api/src/lib/__tests__/derivations-csv.test.ts @@ -21,6 +21,8 @@ import { createTestLocation, createTestMicronixPlate, createTestMicronixTube, + createTestCryovialBox, + createTestCryovialTube, createTestBag, createTestSpecimen, createTestStorageContainer, @@ -361,6 +363,42 @@ describe('derivations-csv', () => { expect(result.rows[2].error).toMatch(/more than once in your file/) }) + /** A cryovial tube with `barcode`, on the parent's Specimen. */ + async function addCryovial(seed: Awaited>, barcode: string) { + const cryoBox = await createTestCryovialBox(testDb, { name: 'BoxDef', locationId: seed.loc.id }) + const [sc] = await testDb + .insert(storageContainer) + .values({ specimenId: seed.parent.specimenId, unitId: seed.unit.id, totalQuantity: 5, remainingQuantity: 5 }) + .returning() + await createTestCryovialTube(testDb, { containerId: sc.id, collectionId: cryoBox.id, barcode, position: 'A01' }) + return sc + } + + it('rejects a row whose parent barcode is on several Containers instead of drawing from one', async () => { + const seed = await seedParent() + const cryovial = await addCryovial(seed, 'MT-PARENT') + const csv = ['parent_container_barcode,plate_name,position,container_barcode', 'MT-PARENT,PlateDef,B01,C1'].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, settings) + + expect(result.rows[0].valid).toBe(false) + expect(result.rows[0].error).toBe( + `Ambiguous barcode 'MT-PARENT': it matches 2 containers (container ${seed.parent.id} (micronix_tube in PlateDef A01), ` + + `container ${cryovial.id} (cryovial_tube in BoxDef A01)). Identify the container by its id or position instead.`, + ) + }) + + it('rejects a child barcode another tube type already has, ignoring case', async () => { + const seed = await seedParent() + await addCryovial(seed, 'cv-777') + const csv = [header, `${seed.parent.id},PlateDef,B01,CV-777`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, settings) + + expect(result.rows[0].valid).toBe(false) + expect(result.rows[0].error).toMatch(/already used for another container/) + }) + it('warns about a mismatched source type even when the first row has no parent', async () => { const { parent, plate, unit } = await seedParent() const plasmaType = await createTestSpecimenType(testDb, { name: 'Plasma' }) diff --git a/packages/api/src/lib/__tests__/identifier-resolution.test.ts b/packages/api/src/lib/__tests__/identifier-resolution.test.ts index dfa88e61..a412f5c8 100644 --- a/packages/api/src/lib/__tests__/identifier-resolution.test.ts +++ b/packages/api/src/lib/__tests__/identifier-resolution.test.ts @@ -5,12 +5,26 @@ import { resolveSubjectByNameAndStudy, resolveSpecimenTypeByName, resolveContainerByBarcode, + findContainersByBarcode, + findTubesUsingBarcode, + AmbiguousBarcodeError, resolveStudiesByShortCodes, resolveSpecimenTypesByNames, resolveSubjectNamesByStudy, } from '../identifier-resolution' import { SQL_IN_CHUNK_SIZE } from '../sql-helpers' -import { createTestStudy, createTestStudySubject, createTestSpecimenType, createTestStorageContainer, createTestStorageType, createTestLocation } from '../../__tests__/helpers/factories' +import { + createTestStudy, + createTestStudySubject, + createTestSpecimenType, + createTestStorageContainer, + createTestStorageType, + createTestLocation, + createTestCryovialBox, + createTestCryovialTube, + createTestMicronixPlate, + createTestMicronixTube, +} from '../../__tests__/helpers/factories' import { box, sheet, paper } from '../../db/schema' import { utcNow } from '../datetime' import type { Database } from '../../db/client' @@ -104,11 +118,69 @@ describe('identifier-resolution lib', () => { sublabel: 'SPOT-42', }) - const id = await resolveContainerByBarcode(testDb, 'SPOT-42') - expect(id).toBe(container.id) + const match = await resolveContainerByBarcode(testDb, 'SPOT-42') + expect(match?.containerId).toBe(container.id) + }) + + it('throws AmbiguousBarcodeError listing every Container when the barcode is on several', async () => { + const { first, second } = await seedSharedCryovialBarcode('NA') + + const error = await resolveContainerByBarcode(testDb, 'NA').catch((e: unknown) => e) + + expect(error).toBeInstanceOf(AmbiguousBarcodeError) + expect((error as AmbiguousBarcodeError).candidates.map((c) => c.containerId)).toEqual([first, second]) + expect((error as AmbiguousBarcodeError).message).toBe( + `Ambiguous barcode 'NA': it matches 2 containers (container ${first} (cryovial_tube in BOX-1 A01), ` + + `container ${second} (cryovial_tube in BOX-1 A02)). Identify the container by its id or position instead.`, + ) + }) + }) + + describe('findContainersByBarcode', () => { + it('returns every Container carrying the barcode, ordered by id', async () => { + const { first, second } = await seedSharedCryovialBarcode('NA') + const matches = await findContainersByBarcode(testDb, 'NA') + expect(matches.map((m) => [m.containerId, m.containerType, m.barcode])).toEqual([ + [first, 'cryovial_tube', 'NA'], + [second, 'cryovial_tube', 'NA'], + ]) + }) + + it('matches exactly unless asked to ignore case', async () => { + await seedSharedCryovialBarcode('CK3-3FR4') + expect(await findContainersByBarcode(testDb, 'Ck3-3Fr4')).toEqual([]) + expect(await findContainersByBarcode(testDb, ' Ck3-3Fr4 ', { ignoreCase: true })).toHaveLength(2) }) }) + describe('findTubesUsingBarcode', () => { + it('finds tubes of every type ignoring case, except the given Container', async () => { + const { location, sibling } = await seedSharedCryovialBarcode('OTHER') + const plate = await createTestMicronixPlate(testDb, { name: 'PLATE-1', locationId: location.id }) + const micronix = await createTestStorageContainer(testDb, sibling) + await createTestMicronixTube(testDb, { containerId: micronix.id, collectionId: plate.id, barcode: 'MX-1' }) + const box2 = await createTestCryovialBox(testDb, { name: 'BOX-2', locationId: location.id }) + const cryovial = await createTestStorageContainer(testDb, sibling) + await createTestCryovialTube(testDb, { containerId: cryovial.id, collectionId: box2.id, barcode: 'mx-1 ' }) + + expect((await findTubesUsingBarcode(testDb, 'Mx-1')).map((m) => m.containerId)).toEqual([micronix.id, cryovial.id]) + expect((await findTubesUsingBarcode(testDb, 'MX-1', micronix.id)).map((m) => m.containerId)).toEqual([cryovial.id]) + }) + }) + + /** Two cryovial tubes in BOX-1 (A01 and A02) whose barcodes are `barcode` and a lower-case copy when it has letters. */ + async function seedSharedCryovialBarcode(barcode: string) { + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Freezer 1', storageTypeId: String(storageType.id), path: 'Freezer 1' }) + const cryoBox = await createTestCryovialBox(testDb, { name: 'BOX-1', locationId: location.id }) + const a = await createTestStorageContainer(testDb, {}) + const b = await createTestStorageContainer(testDb, { unitId: a.unitId, specimenId: a.specimenId }) + const second = barcode === 'CK3-3FR4' ? barcode.toLowerCase() : barcode + await createTestCryovialTube(testDb, { containerId: a.id, collectionId: cryoBox.id, barcode, position: 'A01' }) + await createTestCryovialTube(testDb, { containerId: b.id, collectionId: cryoBox.id, barcode: second, position: 'A02' }) + return { first: a.id, second: b.id, location, sibling: { unitId: a.unitId, specimenId: a.specimenId } } + } + describe('resolveStudiesByShortCodes', () => { it('returns empty map for empty array', async () => { const map = await resolveStudiesByShortCodes(testDb, []) diff --git a/packages/api/src/lib/container-move.ts b/packages/api/src/lib/container-move.ts index 15138b9d..8b9d517c 100644 --- a/packages/api/src/lib/container-move.ts +++ b/packages/api/src/lib/container-move.ts @@ -14,6 +14,7 @@ import { eq, and, isNull } from 'drizzle-orm' import { resolveCollection, resolveSheetsByName, type CollectionType } from './collections/collection-resolve' import { checkGridPositionOccupancy } from './container-occupancy' import { normalizePosition } from './normalize-position' +import { AmbiguousBarcodeError, resolveContainerByBarcode } from './identifier-resolution' import type { ContainerSubtype } from './container-placement' /** Staging prefix for position updates so swaps on one plate do not violate UNIQUE(collection_id, position) mid-transaction. */ @@ -153,55 +154,6 @@ export async function resolveContainerByPosition( return null } -/** - * Resolve container by barcode - */ -export async function resolveContainerByBarcode(database: Database, barcode: string): Promise { - const micronix = await database.select().from(micronixTube).where(eq(micronixTube.barcode, barcode)).get() - if (micronix) { - const plate = await database.select({ name: micronixPlate.name }).from(micronixPlate).where(eq(micronixPlate.id, micronix.collectionId)).get() - return { - containerId: micronix.id, - containerType: 'micronix_tube', - currentCollectionId: micronix.collectionId, - currentCollectionName: plate?.name || null, - currentCollectionType: 'micronix_plate', - currentPosition: micronix.position || null, - barcode: micronix.barcode, - } - } - - const cryovial = await database.select().from(cryovialTube).where(eq(cryovialTube.barcode, barcode)).get() - if (cryovial) { - const boxRec = await database.select({ name: cryovialBox.name }).from(cryovialBox).where(eq(cryovialBox.id, cryovial.collectionId)).get() - return { - containerId: cryovial.id, - containerType: 'cryovial_tube', - currentCollectionId: cryovial.collectionId, - currentCollectionName: boxRec?.name || null, - currentCollectionType: 'cryovial_box', - currentPosition: cryovial.position || null, - barcode: cryovial.barcode || null, - } - } - - const paperRec = await database.select().from(paper).where(eq(paper.sublabel, barcode)).get() - if (paperRec) { - const sheetRec = await database.select({ name: sheet.name }).from(sheet).where(eq(sheet.id, paperRec.sheetId)).get() - return { - containerId: paperRec.id, - containerType: 'paper', - currentCollectionId: paperRec.sheetId, - currentCollectionName: sheetRec?.name || null, - currentCollectionType: 'sheet', - currentPosition: null, - barcode: paperRec.sublabel || null, - } - } - - return null -} - /** * Check position availability */ @@ -242,7 +194,15 @@ export interface ContainerIdentifier { */ export async function resolveContainerByContainerId(database: Database, containerId: number): Promise { const micronix = await database.select().from(micronixTube).where(eq(micronixTube.id, containerId)).get() - if (micronix) return resolveContainerByBarcode(database, micronix.barcode) + if (micronix) return { + containerId: micronix.id, + containerType: 'micronix_tube', + currentCollectionId: micronix.collectionId, + currentCollectionName: (await database.select({ name: micronixPlate.name }).from(micronixPlate).where(eq(micronixPlate.id, micronix.collectionId)).get())?.name || null, + currentCollectionType: 'micronix_plate', + currentPosition: micronix.position, + barcode: micronix.barcode, + } const cryovial = await database.select().from(cryovialTube).where(eq(cryovialTube.id, containerId)).get() if (cryovial) return { @@ -281,7 +241,8 @@ export async function resolveContainerByContainerId(database: Database, containe } /** - * Resolve container by identifier + * Resolve container by identifier. Throws `AmbiguousBarcodeError` when a barcode matches + * more than one Container. */ export async function resolveContainerByIdentifier(database: Database, identifier: ContainerIdentifier): Promise { if (identifier.type === 'barcode' && identifier.barcode) { @@ -298,23 +259,28 @@ export async function resolveContainerByIdentifier(database: Database, identifie return null } -export async function resolveContainersByIdentifiers(database: Database, identifiers: ContainerIdentifier[]): Promise> { - const results = new Map() +export function containerIdentifierKey(identifier: ContainerIdentifier): string { + return identifier.type === 'barcode' ? identifier.barcode! : identifier.type === 'position' ? `${identifier.sourceCollectionName}:${identifier.sourcePosition}` : `container_${identifier.containerId}` +} + +/** What each identifier resolved to, by `containerIdentifierKey`; identifiers that match nothing are absent. */ +export async function resolveContainersByIdentifiers( + database: Database, + identifiers: ContainerIdentifier[], +): Promise> { + const results = new Map() for (const identifier of identifiers) { - const container = await resolveContainerByIdentifier(database, identifier) - if (container) { - const key = identifier.type === 'barcode' ? identifier.barcode! : identifier.type === 'position' ? `${identifier.sourceCollectionName}:${identifier.sourcePosition}` : `container_${identifier.containerId}` - results.set(key, container) + try { + const container = await resolveContainerByIdentifier(database, identifier) + if (container) results.set(containerIdentifierKey(identifier), { container }) + } catch (error) { + if (!(error instanceof AmbiguousBarcodeError)) throw error + results.set(containerIdentifierKey(identifier), { ambiguous: error }) } } return results } -export async function resolveContainersByBarcodes(database: Database, barcodes: string[]): Promise> { - const identifiers: ContainerIdentifier[] = barcodes.map(b => ({ type: 'barcode', barcode: b })) - return resolveContainersByIdentifiers(database, identifiers) -} - export function inferCollectionTypeFromContainers(containers: ContainerInfo[]): { valid: boolean; collectionType: CollectionType | null; error?: string } { if (containers.length === 0) return { valid: false, collectionType: null, error: 'No containers provided' } const collectionTypes = new Set() @@ -336,11 +302,15 @@ export async function executeMoves(database: Database, request: BatchMoveRequest })) const errors: ValidationError[] = [] - const resolvedMoves = await Promise.all(moves.map(async (move, index) => ({ - row: index + 1, - move, - info: await resolveContainerByIdentifier(database, move.identifier), - }))) + const resolvedMoves = await Promise.all(moves.map(async (move, index) => { + try { + return { row: index + 1, move, info: await resolveContainerByIdentifier(database, move.identifier), error: 'Container not found' } + } catch (error) { + // A barcode on several Containers fails its row instead of moving whichever comes first. + if (!(error instanceof AmbiguousBarcodeError)) throw error + return { row: index + 1, move, info: null, error: error.message } + } + })) const validResolvedMoves: Array<{ row: number @@ -351,7 +321,7 @@ export async function executeMoves(database: Database, request: BatchMoveRequest for (const resolved of resolvedMoves) { if (!resolved.info) { - errors.push({ row: resolved.row, error: 'Container not found' }) + errors.push({ row: resolved.row, error: resolved.error }) continue } validResolvedMoves.push({ diff --git a/packages/api/src/lib/data-audit.ts b/packages/api/src/lib/data-audit.ts index 6311008c..93668720 100644 --- a/packages/api/src/lib/data-audit.ts +++ b/packages/api/src/lib/data-audit.ts @@ -322,9 +322,13 @@ export interface StorageContainerTagOrphanItem { tagId: number } +/** Tubes whose barcodes are equal ignoring case and surrounding whitespace. */ export interface DuplicateBarcodeItem { + /** The barcode as most of the group spells it. */ barcode: string - containerType: 'micronix_tube' + /** Every stored spelling in the group; more than one means case or whitespace variants. */ + variants: string[] + containerTypes: Array<'micronix_tube' | 'cryovial_tube'> ids: number[] } @@ -456,20 +460,48 @@ export async function listStorageContainerTagOrphans(db: Database): Promise ({ storageContainerId: l.storageContainerId, tagId: l.tagId })) } -/** Barcodes that appear more than once in micronix_tube (unique in schema; duplicate = integrity issue). Cryovial barcodes are not unique and are not treated as an integrity issue. */ +/** + * Tube barcodes on more than one tube, across micronix and cryovial tubes, comparing trimmed + * and case-folded values so `CK3-3FR4` and `ck3-3fr4 ` count as one barcode. Barcode lookups + * refuse to choose between such tubes, and new or changed tube barcodes can't add to them. + */ export async function listDuplicateBarcodes(db: Database): Promise { - const results: DuplicateBarcodeItem[] = [] - const micronixRows = await db.select({ id: micronixTube.id, barcode: micronixTube.barcode }).from(micronixTube) - const micronixByBarcode = new Map() - for (const row of micronixRows) { - const list = micronixByBarcode.get(row.barcode) ?? [] - list.push(row.id) - micronixByBarcode.set(row.barcode, list) + const rows = [ + ...(await db.select({ id: micronixTube.id, barcode: micronixTube.barcode }).from(micronixTube)).map((row) => ({ + ...row, + containerType: 'micronix_tube' as const, + })), + ...(await db + .select({ id: cryovialTube.id, barcode: cryovialTube.barcode }) + .from(cryovialTube) + .where(sql`${cryovialTube.barcode} IS NOT NULL AND trim(${cryovialTube.barcode}) != ''`)).map((row) => ({ + id: row.id, + barcode: row.barcode ?? '', + containerType: 'cryovial_tube' as const, + })), + ] + const groups = new Map() + for (const row of rows) { + const key = row.barcode.trim().toLowerCase() + const group = groups.get(key) ?? [] + group.push(row) + groups.set(key, group) } - for (const [barcode, ids] of micronixByBarcode) { - if (ids.length > 1) results.push({ barcode, containerType: 'micronix_tube', ids }) + const results: DuplicateBarcodeItem[] = [] + for (const group of groups.values()) { + if (group.length < 2) continue + const spellings = new Map() + for (const row of group) spellings.set(row.barcode, (spellings.get(row.barcode) ?? 0) + 1) + const variants = [...spellings.keys()].sort() + const barcode = [...spellings.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))[0][0] + results.push({ + barcode, + variants, + containerTypes: [...new Set(group.map((row) => row.containerType))].sort(), + ids: group.map((row) => row.id).sort((a, b) => a - b), + }) } - return results + return results.sort((a, b) => a.barcode.localeCompare(b.barcode)) } /** Locations whose stored path does not match the path computed from parent chain. */ diff --git a/packages/api/src/lib/derivations-csv.ts b/packages/api/src/lib/derivations-csv.ts index 119a86e1..e73f5631 100644 --- a/packages/api/src/lib/derivations-csv.ts +++ b/packages/api/src/lib/derivations-csv.ts @@ -4,7 +4,6 @@ import { cryovialBox, cryovialTube, micronixPlate, - micronixTube, paper, specimen, specimenType, @@ -23,6 +22,7 @@ import { import { createDerivation, type CreateDerivationInput } from './derivations' import { withWriteTransaction } from '../db/write-transaction' import { resolveParentContainerId } from './derivations/parent-container-resolver' +import { findTubesUsingBarcode } from './identifier-resolution' import { normalizePosition } from './normalize-position' import { checkGridPositionOccupancy } from './container-occupancy' import { CollectionPlanner } from './specimen-registration/collections' @@ -379,8 +379,8 @@ export async function validateDerivationsCsv( const collectionsMap = new Map() /** Track (collectionKey, position) to catch duplicate position in same collection within the CSV */ const seenCollectionPosition = new Set() - /** Track micronix container_barcode to catch duplicates within the CSV */ - const seenMicronixBarcode = new Set() + /** Track tube container_barcode (lowercased) to catch duplicates within the CSV */ + const seenTubeBarcode = new Set() let validCount = 0 let invalidCount = 0 let warningCount = 0 @@ -567,35 +567,31 @@ export async function validateDerivationsCsv( } // For micronix tubes, container_barcode is required (barcodes are scanned and provided externally) - if (containerType === 'micronix_tube') { + if (containerType === 'micronix_tube' || containerType === 'cryovial_tube') { const barcode = (row.container_barcode ?? '').toString().trim() - if (!barcode) { + if (!barcode && containerType === 'micronix_tube') { validationRow.error = 'container_barcode is required for micronix tube derivations. Barcodes are scanned and provided by you; the system does not assign them.' validationRows.push(validationRow) invalidCount++ continue } - if (seenMicronixBarcode.has(barcode)) { + // Tube barcodes are unique across tube types, ignoring case. + if (barcode && seenTubeBarcode.has(barcode.toLowerCase())) { validationRow.error = - 'That micronix barcode is used more than once in your file. Each barcode must be unique.' + 'That tube barcode is used more than once in your file. Each barcode must be unique.' validationRows.push(validationRow) invalidCount++ continue } - const existingTube = await database - .select({ id: micronixTube.id }) - .from(micronixTube) - .where(eq(micronixTube.barcode, barcode)) - .get() - if (existingTube) { + if (barcode && (await findTubesUsingBarcode(database, barcode)).length > 0) { validationRow.error = 'That barcode is already used for another container. Use a different barcode. Barcodes are scanned and provided by you; the system does not assign them.' validationRows.push(validationRow) invalidCount++ continue } - seenMicronixBarcode.add(barcode) + if (barcode) seenTubeBarcode.add(barcode.toLowerCase()) } if (containerType === 'paper') { diff --git a/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts b/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts index ed8c8110..cb7d1e78 100644 --- a/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts +++ b/packages/api/src/lib/derivations/__tests__/parent-container-resolver.test.ts @@ -77,6 +77,21 @@ describe('resolveParentContainerId', () => { expect(id).toBe(container.id) }) + it('refuses a barcode on several Containers and lists them, instead of picking one', async () => { + const location = await freezer(testDb) + const now = utcNow() + const [boxRow] = await testDb.insert(cryovialBox).values({ name: 'Box-NA', locationId: location.id, created: now, lastUpdated: now }).returning() + const first = await createTestStorageContainer(testDb) + const second = await createTestStorageContainer(testDb, { unitId: first.unitId, specimenId: first.specimenId }) + await testDb.insert(cryovialTube).values({ id: first.id, collectionId: boxRow!.id, barcode: 'NA', position: 'A01' }) + await testDb.insert(cryovialTube).values({ id: second.id, collectionId: boxRow!.id, barcode: 'NA', position: 'A02' }) + + await expect(resolveParentContainerId(testDb, row({ parent_container_barcode: ' NA ' }))).rejects.toThrow( + `Ambiguous barcode 'NA': it matches 2 containers (container ${first.id} (cryovial_tube in Box-NA A01), ` + + `container ${second.id} (cryovial_tube in Box-NA A02)). Identify the container by its id or position instead.`, + ) + }) + it('throws when the barcode is not found', async () => { await expect(resolveParentContainerId(testDb, row({ parent_container_barcode: 'MISSING' }))).rejects.toThrow( "Parent container barcode 'MISSING' not found", diff --git a/packages/api/src/lib/derivations/parent-container-resolver.ts b/packages/api/src/lib/derivations/parent-container-resolver.ts index faa446ef..fd087b08 100644 --- a/packages/api/src/lib/derivations/parent-container-resolver.ts +++ b/packages/api/src/lib/derivations/parent-container-resolver.ts @@ -25,7 +25,7 @@ function parseNumber(value?: string): number | undefined { * * Parents may be identified four ways, tried in order: * 1. explicit `parent_container_id` - * 2. `parent_container_barcode` (any container subtype) + * 2. `parent_container_barcode` (any container subtype; a barcode on several Containers is an error) * 3. control-batch provenance (`parent_control_batch_name` / `_id` + specimen type) * 4. study-subject provenance (`parent_study_short_code` + `parent_subject_name` + specimen type) * @@ -52,8 +52,9 @@ export async function resolveParentContainerId( // 2. Barcode across known subtypes if (row.parent_container_barcode) { const barcode = row.parent_container_barcode.trim() - const containerId = await resolveContainerByBarcode(database, barcode) - if (containerId != null) return containerId + // Throws AmbiguousBarcodeError rather than draw quantity from whichever match comes first. + const match = await resolveContainerByBarcode(database, barcode) + if (match) return match.containerId throw new Error(`Parent container barcode '${barcode}' not found`) } diff --git a/packages/api/src/lib/identifier-resolution.ts b/packages/api/src/lib/identifier-resolution.ts index 35cc2587..7792b9e7 100644 --- a/packages/api/src/lib/identifier-resolution.ts +++ b/packages/api/src/lib/identifier-resolution.ts @@ -1,6 +1,18 @@ import type { Database } from '../db/client' -import { study, studySubject, specimenType, micronixTube, storageContainer, cryovialTube, paper } from '../db/schema' -import { eq, and, inArray } from 'drizzle-orm' +import { + study, + studySubject, + specimenType, + micronixTube, + micronixPlate, + cryovialTube, + cryovialBox, + paper, + sheet, +} from '../db/schema' +import { eq, and, inArray, sql } from 'drizzle-orm' +import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' +import { ValidationError } from './error-handler' import type { DatabaseOrTransaction } from './db-types' import { chunkArray, SQL_IN_CHUNK_SIZE } from './sql-helpers' @@ -52,35 +64,142 @@ export async function resolveSpecimenTypeByName(database: Database, name: string return specimenTypeRecord?.id ?? null } +/** A Container a barcode lookup matched. Paper matches on its sublabel. */ +export interface BarcodeMatch { + containerId: number + containerType: 'micronix_tube' | 'cryovial_tube' | 'paper' + /** The stored barcode (or sublabel), which may differ in case or whitespace under `ignoreCase`. */ + barcode: string + currentCollectionId: number + currentCollectionName: string | null + currentCollectionType: 'micronix_plate' | 'cryovial_box' | 'sheet' + currentPosition: string | null +} + +export interface FindContainersByBarcodeOptions { + /** Compare trimmed and case-folded, as tube barcode uniqueness does. Default: exact match. */ + ignoreCase?: boolean + /** Match micronix and cryovial tube barcodes only, not Paper sublabels. */ + tubesOnly?: boolean +} + /** - * Resolve container barcode to container ID + * Every Container whose barcode (or Paper sublabel) is `barcode`, ordered by Container id. + * Barcodes aren't unique across tube types, and cryovial barcodes aren't unique at all, so + * a caller that needs one Container must handle several matches, for example with + * `requireSingleBarcodeMatch`. */ -export async function resolveContainerByBarcode(database: DatabaseOrTransaction, barcode: string): Promise { - // Try micronix tube +export async function findContainersByBarcode( + database: DatabaseOrTransaction, + barcode: string, + options: FindContainersByBarcodeOptions = {}, +): Promise { + const matches = (column: SQLiteColumn) => + options.ignoreCase ? sql`lower(trim(${column})) = lower(${barcode.trim()})` : eq(column, barcode) + const micronix = await database - .select({ id: micronixTube.id }) + .select({ + containerId: micronixTube.id, + barcode: micronixTube.barcode, + currentCollectionId: micronixTube.collectionId, + currentCollectionName: micronixPlate.name, + currentPosition: micronixTube.position, + }) .from(micronixTube) - .where(eq(micronixTube.barcode, barcode)) - .get() - if (micronix) return micronix.id - - // Try cryovial tube + .leftJoin(micronixPlate, eq(micronixPlate.id, micronixTube.collectionId)) + .where(matches(micronixTube.barcode)) const cryovial = await database - .select({ id: cryovialTube.id }) + .select({ + containerId: cryovialTube.id, + barcode: cryovialTube.barcode, + currentCollectionId: cryovialTube.collectionId, + currentCollectionName: cryovialBox.name, + currentPosition: cryovialTube.position, + }) .from(cryovialTube) - .where(eq(cryovialTube.barcode, barcode)) - .get() - if (cryovial) return cryovial.id + .leftJoin(cryovialBox, eq(cryovialBox.id, cryovialTube.collectionId)) + .where(matches(cryovialTube.barcode)) + const papers = options.tubesOnly + ? [] + : await database + .select({ + containerId: paper.id, + barcode: paper.sublabel, + currentCollectionId: paper.sheetId, + currentCollectionName: sheet.name, + }) + .from(paper) + .leftJoin(sheet, eq(sheet.id, paper.sheetId)) + .where(matches(paper.sublabel)) - // Try paper - const paperRec = await database - .select({ id: paper.id }) - .from(paper) - .where(eq(paper.sublabel, barcode)) - .get() - if (paperRec) return paperRec.id - - return null + const all: BarcodeMatch[] = [ + ...micronix.map((row) => ({ ...row, containerType: 'micronix_tube' as const, currentCollectionType: 'micronix_plate' as const })), + ...cryovial.map((row) => ({ + ...row, + barcode: row.barcode ?? '', + containerType: 'cryovial_tube' as const, + currentCollectionType: 'cryovial_box' as const, + })), + ...papers.map((row) => ({ + ...row, + barcode: row.barcode ?? '', + currentPosition: null, + containerType: 'paper' as const, + currentCollectionType: 'sheet' as const, + })), + ] + return all.sort((a, b) => a.containerId - b.containerId) +} + +/** How many candidates an ambiguous-barcode message names before summarising the rest. */ +const LISTED_CANDIDATES = 10 + +function describeBarcodeMatch(match: BarcodeMatch): string { + const place = [match.currentCollectionName, match.currentPosition].filter(Boolean).join(' ') + return `container ${match.containerId} (${match.containerType}${place ? ` in ${place}` : ''})` +} + +/** + * A barcode lookup matched more than one Container and won't guess between them. `details` + * lists every candidate; the message names the first few. + */ +export class AmbiguousBarcodeError extends ValidationError { + constructor( + readonly barcode: string, + readonly candidates: BarcodeMatch[], + ) { + const listed = candidates.slice(0, LISTED_CANDIDATES).map(describeBarcodeMatch).join(', ') + const more = candidates.length > LISTED_CANDIDATES ? `, and ${candidates.length - LISTED_CANDIDATES} more` : '' + super( + `Ambiguous barcode '${barcode}': it matches ${candidates.length} containers (${listed}${more}). ` + + 'Identify the container by its id or position instead.', + { code: 'AMBIGUOUS_BARCODE', barcode, candidates }, + ) + this.name = 'AmbiguousBarcodeError' + } +} + +/** The one Container `barcode` names, or null when none does. Throws `AmbiguousBarcodeError` for several. */ +export async function resolveContainerByBarcode( + database: DatabaseOrTransaction, + barcode: string, +): Promise { + const matches = await findContainersByBarcode(database, barcode) + if (matches.length > 1) throw new AmbiguousBarcodeError(barcode, matches) + return matches[0] ?? null +} + +/** + * Tubes (micronix or cryovial) whose barcode equals `barcode` ignoring case and surrounding + * whitespace, other than `exceptContainerId`. A new or changed tube barcode must match none. + */ +export async function findTubesUsingBarcode( + database: DatabaseOrTransaction, + barcode: string, + exceptContainerId?: number, +): Promise { + const matches = await findContainersByBarcode(database, barcode, { ignoreCase: true, tubesOnly: true }) + return matches.filter((match) => match.containerId !== exceptContainerId) } /** diff --git a/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts b/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts index 34100513..e07d3c1f 100644 --- a/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts +++ b/packages/api/src/lib/specimen-registration/__tests__/specimen-registration.test.ts @@ -302,6 +302,48 @@ describe('Specimen registration', () => { ) }) + describe('tube barcodes across types', () => { + const cryovial = (barcode: string, position: string): RegistrationContainer => ({ + containerType: 'cryovial_tube', + barcode, + collection: { type: 'cryovial_box', name: 'CB1', locationId, position }, + }) + + it('rejects a micronix barcode a cryovial tube already has, ignoring case', async () => { + await registerSpecimens(db, [entry([cryovial('CV-9', 'A01')])]) + + const error = await rejection([entry([tube('cv-9', { name: 'P1', position: 'A01' })])]) + + expect(error.errors).toEqual([ + { entryIndex: 0, containerIndex: 0, field: 'barcode', message: "Barcode 'cv-9' already exists as 'CV-9'. Use a different barcode." }, + ]) + expect(await count(storageContainer)).toBe(1) + }) + + it('rejects a cryovial barcode a micronix tube already has', async () => { + await registerSpecimens(db, [entry([tube('MX-9', { name: 'P1', position: 'A01' })])]) + + const error = await rejection([entry([cryovial('MX-9', 'A01')])]) + + expect(error.errors).toEqual([ + { entryIndex: 0, containerIndex: 0, field: 'barcode', message: "Barcode 'MX-9' already exists. Use a different barcode." }, + ]) + }) + + it('rejects one barcode on two tube types in the same request', async () => { + const error = await rejection([entry([tube('X1', { name: 'P1', position: 'A01' }), cryovial('x1', 'A01')])]) + + expect(error.errors).toEqual([ + { + entryIndex: 0, + containerIndex: 1, + field: 'barcode', + message: "Barcode 'x1' is used more than once in this request. Each barcode must be unique.", + }, + ]) + }) + }) + it('keys a missing position to the position field', async () => { const error = await rejection([entry([tube('T1', { name: 'P1' })])]) diff --git a/packages/api/src/lib/specimen-registration/plan.ts b/packages/api/src/lib/specimen-registration/plan.ts index 5f9e07e2..29b7f96c 100644 --- a/packages/api/src/lib/specimen-registration/plan.ts +++ b/packages/api/src/lib/specimen-registration/plan.ts @@ -20,7 +20,8 @@ import { validateContainerTypeForSpecimenType, validateUnitForContainerType, } from '../validation' -import { GRID_COLLECTION_OF, TUBE_TABLES } from './tables' +import { GRID_COLLECTION_OF } from './tables' +import { findTubesUsingBarcode } from '../identifier-resolution' import { CollectionPlanner, targetKey, type CollectionTarget, type GridCollectionType, type PlannedCollection } from './collections' import { validateContainerWriteFields } from './container-fields' import type { RegistrationContainer, RegistrationEntry, RegistrationError, RegistrationSource } from './types' @@ -88,14 +89,15 @@ function specimenIdentityKey(identity: SpecimenIdentity): string { /** Something a request may use only once. */ type Claim = - | { kind: 'barcode'; containerType: 'micronix_tube' | 'cryovial_tube'; barcode: string } + | { kind: 'barcode'; barcode: string } | { kind: 'position'; gridType: GridCollectionType; collection: CollectionTarget; position: string } | { kind: 'sublabel'; sheet: CollectionTarget; sublabel: string } function claimKey(claim: Claim): string { switch (claim.kind) { case 'barcode': - return `barcode:${claim.containerType}:${claim.barcode}` + // Tube barcodes are unique across types and case. + return `barcode:${claim.barcode.toLowerCase()}` case 'position': return `position:${claim.gridType}:${targetKey(claim.collection)}:${claim.position}` case 'sublabel': @@ -305,10 +307,13 @@ async function planContainer( const barcode = 'barcode' in input ? input.barcode?.trim() : undefined if (barcode && (input.containerType === 'micronix_tube' || input.containerType === 'cryovial_tube')) { - const table = TUBE_TABLES[input.containerType] - const taken = await database.select({ id: table.id }).from(table).where(eq(table.barcode, barcode)).get() - if (taken) problems.push({ field: 'barcode', message: `Barcode '${barcode}' already exists. Use a different barcode.` }) - if (!claims.take({ kind: 'barcode', containerType: input.containerType, barcode })) { + // A tube barcode must be new across every tube type, ignoring case and whitespace. + const taken = await findTubesUsingBarcode(database, barcode) + if (taken.length > 0) { + const asVariant = taken.some((match) => match.barcode === barcode) ? '' : ` as '${taken[0].barcode}'` + problems.push({ field: 'barcode', message: `Barcode '${barcode}' already exists${asVariant}. Use a different barcode.` }) + } + if (!claims.take({ kind: 'barcode', barcode })) { problems.push({ field: 'barcode', message: `Barcode '${barcode}' is used more than once in ${claims.inputName}. Each barcode must be unique.`, diff --git a/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts b/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts index 5c25eabd..f059576a 100644 --- a/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts +++ b/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts @@ -8,6 +8,12 @@ import { createTestSpecimenType, createTestStudy, createTestStudySubject, + createTestStorageType, + createTestLocation, + createTestCryovialBox, + createTestCryovialTube, + createTestStorageContainer, + createTestUnit, } from '../../../__tests__/helpers/factories' import { listSpecimens } from '../specimen-read' @@ -53,6 +59,32 @@ describe('specimen-read', () => { expect(result.specimens.every((s) => s.study?.shortCode === 'STYA')).toBe(true) }) + it('filters by barcode to every Specimen with a Container carrying it', async () => { + const study = await createTestStudy(testDb, { title: 'NA Study', shortCode: 'NAS' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj NA' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Freezer 1', storageTypeId: String(storageType.id) }) + const cryoBox = await createTestCryovialBox(testDb, { name: 'BOX-NA', locationId: location.id }) + const specimens = [] + for (const [index, date] of ['2024-01-01', '2024-02-01', '2024-03-01'].entries()) { + const s = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id, collectionDate: date }) + specimens.push(s) + const container = await createTestStorageContainer(testDb, { specimenId: s.id, unitId: unit.id }) + await createTestCryovialTube(testDb, { + containerId: container.id, + collectionId: cryoBox.id, + barcode: index < 2 ? 'NA' : 'OTHER', + position: `A0${index + 1}`, + }) + } + + const result = await listSpecimens(testDb, { barcode: 'NA' }) + + expect(result.specimens.map((s) => s.id).sort((a, b) => a - b)).toEqual([specimens[0].id, specimens[1].id]) + }) + it('returns empty result with default pagination for unknown barcode', async () => { const result = await listSpecimens(testDb, { barcode: 'NONEXISTENT-BARCODE' }) expect(result.specimens).toEqual([]) diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 327b24a9..7ba57eb0 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -1,8 +1,8 @@ import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject, study, specimenType, controlBatch } from '../../db/schema' -import { eq, and, or, sql } from 'drizzle-orm' +import { eq, and, or, sql, inArray } from 'drizzle-orm' import { validatePage, validateLimit } from '../constants' -import { resolveContainerByBarcode } from '../identifier-resolution' +import { findContainersByBarcode } from '../identifier-resolution' import { dateToUpperBound } from '../sql-helpers' import { likeContains } from '../sql-like' @@ -156,21 +156,18 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer } if (barcode) { - const containerId = await resolveContainerByBarcode(database, barcode) - if (containerId) { - const container = await database - .select({ specimenId: storageContainer.specimenId }) - .from(storageContainer) - .where(eq(storageContainer.id, containerId)) - .get() - if (container) { - conditions.push(eq(specimen.id, container.specimenId)) - } else { - return emptyBarcodeResult() - } - } else { - return emptyBarcodeResult() - } + // Every Specimen with a Container carrying the barcode; cryovial barcodes repeat. + const containerIds = (await findContainersByBarcode(database, barcode)).map((match) => match.containerId) + if (containerIds.length === 0) return emptyBarcodeResult() + conditions.push( + inArray( + specimen.id, + database + .select({ specimenId: storageContainer.specimenId }) + .from(storageContainer) + .where(inArray(storageContainer.id, containerIds)), + ), + ) } if (search) { diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index 7a80c107..b32f98e4 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -17,6 +17,10 @@ import { createTestStudySubject, createTestBox, createTestSheet, + createTestCryovialBox, + createTestCryovialTube, + createTestMicronixTube, + createTestStorageContainer, } from '../../__tests__/helpers/factories' import { setScannerConfigurations } from '../../lib/settings' import { eq } from 'drizzle-orm' @@ -170,6 +174,58 @@ describe('Collections API', () => { }) }) + describe('ambiguous barcodes', () => { + /** A micronix tube on PLATE-A A01 and a cryovial tube in BOX-A A01, both barcoded SHARED. */ + async function seedSharedBarcode() { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'PLATE-A', locationId: loc.id }) + const cryoBox = await createTestCryovialBox(ctx.db, { name: 'BOX-A', locationId: loc.id }) + const micronix = await createTestStorageContainer(ctx.db, {}) + const cryovial = await createTestStorageContainer(ctx.db, { unitId: micronix.unitId, specimenId: micronix.specimenId }) + await createTestMicronixTube(ctx.db, { containerId: micronix.id, collectionId: plate.id, barcode: 'SHARED', position: 'A01' }) + await createTestCryovialTube(ctx.db, { containerId: cryovial.id, collectionId: cryoBox.id, barcode: 'SHARED', position: 'A01' }) + return { micronix: micronix.id, cryovial: cryovial.id } + } + + it('resolve returns no container and lists the candidates for a barcode on several Containers', async () => { + const { micronix, cryovial } = await seedSharedBarcode() + + const res = await ctx.request('/api/collections/containers/resolve', { + method: 'POST', + json: { identifiers: [{ type: 'barcode', barcode: 'SHARED' }] }, + }) + + expect(res.status).toBe(200) + const body = (await res.json()) as { + containers: Array<{ container?: unknown; error: string; candidates: Array<{ containerId: number }> }> + } + expect(body.containers).toHaveLength(1) + expect(body.containers[0].container ?? null).toBeNull() + expect(body.containers[0].error).toMatch(/^Ambiguous barcode 'SHARED': it matches 2 containers/) + expect(body.containers[0].candidates.map((c) => c.containerId)).toEqual([micronix, cryovial]) + }) + + it('move refuses the row instead of moving one of the tubes', async () => { + const { micronix } = await seedSharedBarcode() + + const res = await ctx.request('/api/collections/containers/move', { + method: 'POST', + json: { + collectionType: 'micronix_plate', + mappings: [{ fromCollectionName: 'PLATE-A', toCollectionName: 'PLATE-A' }], + moves: [{ identifier: { type: 'barcode', barcode: 'SHARED' }, targetPosition: 'B01' }], + }, + }) + + expect(res.status).toBe(400) + const body = (await res.json()) as { errors: Array<{ row: number; error: string }> } + expect(body.errors).toEqual([{ row: 1, error: expect.stringMatching(/^Ambiguous barcode 'SHARED'/) }]) + const tube = await ctx.db.select().from(micronixTube).where(eq(micronixTube.id, micronix)).get() + expect(tube?.position).toBe('A01') + }) + }) + describe('POST /api/collections/sheets/move', () => { it('returns a typed 409 when the target box already has a sheet with that name', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index cd9aa39d..0cedaf3c 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -442,6 +442,13 @@ describe('Containers API', () => { return id } + async function seedCryovial(barcode: string, position: string) { + const cryoBox = await createTestCryovialBox(ctx.db, { name: `BOX-${barcode}-${position}`, locationId }) + const id = await seedContainer() + await createTestCryovialTube(ctx.db, { containerId: id, collectionId: cryoBox.id, barcode, position }) + return id + } + async function seedPaper(sublabel: string) { const boxRecord = await createTestBox(ctx.db, { name: `BX-${sublabel}`, locationId }) const sheetRecord = await createTestSheet(ctx.db, { name: 'S1', boxId: boxRecord.id }) @@ -512,7 +519,7 @@ describe('Containers API', () => { const id = await seedMicronix('MX-SELF', 'A01') // The check misses the conflict (as when another request takes the barcode first), so the // UNIQUE constraint fails on the last write of the edit. - vi.spyOn(identifierResolution, 'resolveContainerByBarcode').mockResolvedValue(null) + vi.spyOn(identifierResolution, 'findTubesUsingBarcode').mockResolvedValue([]) const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', @@ -524,6 +531,39 @@ describe('Containers API', () => { await expectUnchanged(id) }) + it('saves a Container whose unchanged barcode other tubes already share', async () => { + await seedCryovial('NA', 'A01') + const id = await seedCryovial('NA', 'A02') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', barcode: 'NA' }, + }) + + expect(res.status).toBe(200) + const row = await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, id)).get() + expect(row?.comment).toBe('changed') + }) + + it('returns 400 for a barcode another tube type has in a different case and leaves the Container unchanged', async () => { + const takenBy = await seedCryovial('CV-1', 'A01') + const id = await seedMicronix('MX-OWN', 'A01') + + const res = await ctx.request(`/api/containers/${id}`, { + method: 'PATCH', + json: { comment: 'changed', remainingQuantity: 0.5, tagIds: [otherTagId], barcode: 'cv-1' }, + }) + + expect(res.status).toBe(400) + expect(await res.json()).toMatchObject({ + error: "Barcode 'cv-1' is already in use as 'CV-1'", + details: { containerIds: [takenBy] }, + }) + await expectUnchanged(id) + const tube = await ctx.db.select().from(micronixTube).where(eq(micronixTube.id, id)).get() + expect(tube?.barcode).toBe('MX-OWN') + }) + it('returns 400 for a Paper sublabel already in use and leaves the Container unchanged', async () => { await seedPaper('SPOT-TAKEN') const id = await seedPaper('SPOT-MINE') diff --git a/packages/api/src/routes/__tests__/data-audit.test.ts b/packages/api/src/routes/__tests__/data-audit.test.ts index 628626f3..8b0a3b7a 100644 --- a/packages/api/src/routes/__tests__/data-audit.test.ts +++ b/packages/api/src/routes/__tests__/data-audit.test.ts @@ -420,29 +420,32 @@ describe('Data audit API', () => { expect(match).toBeDefined() }) - it('does not report duplicate cryovial barcodes as integrity issues', async () => { + it('reports tube barcodes duplicated across types, including case and whitespace variants', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'Shelf', description: '' }) const loc = await createTestLocation(ctx.db, { name: 'LocDup', storageTypeId: String(storageType.id), canContainCollections: true }) const now = utcNow() - const [cvBox1] = await ctx.db.insert(cryovialBox).values({ name: `CryoDup1-${Date.now()}`, locationId: loc.id, created: now, lastUpdated: now }).returning() - const [cvBox2] = await ctx.db.insert(cryovialBox).values({ name: `CryoDup2-${Date.now()}`, locationId: loc.id, created: now, lastUpdated: now }).returning() + const [cvBox] = await ctx.db.insert(cryovialBox).values({ name: 'CryoDup', locationId: loc.id, created: now, lastUpdated: now }).returning() + const [plate] = await ctx.db.insert(micronixPlate).values({ name: 'PlateDup', locationId: loc.id, created: now, lastUpdated: now }).returning() const specType = await createTestSpecimenType(ctx.db, { name: 'STDup' }) - const spec1 = await createTestSpecimen(ctx.db, specType.id) - const unit = await createTestUnit(ctx.db, { symbol: `uL-dup-${Date.now()}`, name: 'microliter', category: 'volume' }) - const c1 = await createTestStorageContainer(ctx.db, { specimenId: spec1.id, unitId: unit.id }) - const spec2 = await createTestSpecimen(ctx.db, specType.id) - const c2 = await createTestStorageContainer(ctx.db, { specimenId: spec2.id, unitId: unit.id }) - const sharedBarcode = `DUP-${Date.now()}` - await ctx.db.insert(cryovialTube).values({ id: c1.id, collectionId: cvBox1!.id, barcode: sharedBarcode, position: null }) - await ctx.db.insert(cryovialTube).values({ id: c2.id, collectionId: cvBox2!.id, barcode: sharedBarcode, position: null }) + const spec = await createTestSpecimen(ctx.db, specType.id) + const unit = await createTestUnit(ctx.db, { symbol: 'uL-dup', name: 'microliter', category: 'volume' }) + const container = () => createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id }) + const [na1, na2, mx, cv, lone] = [await container(), await container(), await container(), await container(), await container()] + await ctx.db.insert(cryovialTube).values({ id: na1.id, collectionId: cvBox!.id, barcode: 'NA', position: null }) + await ctx.db.insert(cryovialTube).values({ id: na2.id, collectionId: cvBox!.id, barcode: 'NA', position: null }) + await ctx.db.insert(micronixTube).values({ id: mx.id, collectionId: plate!.id, barcode: 'CK3-3FR4', position: null }) + await ctx.db.insert(cryovialTube).values({ id: cv.id, collectionId: cvBox!.id, barcode: 'ck3-3fr4 ', position: null }) + await ctx.db.insert(cryovialTube).values({ id: lone.id, collectionId: cvBox!.id, barcode: 'LONE', position: null }) const res = await ctx.request('/api/admin/data-audit/integrity-report', { method: 'GET', }) expect(res.status).toBe(200) - const data = (await res.json()) as { duplicateBarcodes: Array<{ barcode: string; containerType: string }> } - const cryovialMatch = data.duplicateBarcodes.find((d) => d.barcode === sharedBarcode && d.containerType === 'cryovial_tube') - expect(cryovialMatch).toBeUndefined() + const data = (await res.json()) as { duplicateBarcodes: unknown[] } + expect(data.duplicateBarcodes).toEqual([ + { barcode: 'CK3-3FR4', variants: ['CK3-3FR4', 'ck3-3fr4 '], containerTypes: ['cryovial_tube', 'micronix_tube'], ids: [mx.id, cv.id] }, + { barcode: 'NA', variants: ['NA'], containerTypes: ['cryovial_tube'], ids: [na1.id, na2.id] }, + ]) }) it('includes location path inconsistencies in report', async () => { diff --git a/packages/api/src/routes/collections.ts b/packages/api/src/routes/collections.ts index 9f2cebae..a8c0fb35 100644 --- a/packages/api/src/routes/collections.ts +++ b/packages/api/src/routes/collections.ts @@ -1,6 +1,7 @@ import { Hono } from 'hono' import type { Database } from '../db/client' import { executeMoves, type BatchMoveRequest, type ContainerInfo } from '../lib/container-move' +import type { BarcodeMatch } from '../lib/identifier-resolution' import { executeCollectionMoves, type CollectionMoveRequest } from '../lib/collection-move' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { handleRouteError } from '../lib/error-handler' @@ -226,15 +227,28 @@ export function createCollectionsRoutes(database: Database): Hono { const { resolveContainersByIdentifiers } = await import('../lib/container-move') const containers = await resolveContainersByIdentifiers(database, data.identifiers) - const result: Array<{ identifier: unknown; container: ContainerInfo }> = [] - for (const [key, container] of containers.entries()) { + // An ambiguous barcode comes back with no container and the error that lists its candidates. + const result: Array< + | { identifier: unknown; container: ContainerInfo } + | { identifier: unknown; container: null; error: string; candidates: BarcodeMatch[] } + > = [] + for (const [key, resolved] of containers.entries()) { const identifier = data.identifiers.find( (id) => (id.type === 'barcode' && id.barcode === key) || (id.type === 'position' && `${id.sourceCollectionName}:${id.sourcePosition}` === key) || (id.type === 'container_id' && `container_${id.containerId}` === key), ) - result.push({ identifier: identifier || key, container }) + if ('container' in resolved) { + result.push({ identifier: identifier || key, container: resolved.container }) + } else { + result.push({ + identifier: identifier || key, + container: null, + error: resolved.ambiguous.message, + candidates: resolved.ambiguous.candidates, + }) + } } return c.json({ containers: result }) diff --git a/packages/api/src/routes/containers.ts b/packages/api/src/routes/containers.ts index eaeeb44f..73661b6d 100644 --- a/packages/api/src/routes/containers.ts +++ b/packages/api/src/routes/containers.ts @@ -7,7 +7,7 @@ import { validatePage, validateLimit } from '../lib/constants' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' -import { resolveContainerByBarcode } from '../lib/identifier-resolution' +import { findContainersByBarcode, findTubesUsingBarcode } from '../lib/identifier-resolution' import { enrichContainerForApi, enrichContainersForApi, toEnrichedContainerApi } from '../lib/container-api-enrichment' import { loadContainerReadViews } from '../lib/container-read-view' import { mapEnrichedContainerToWire, mapEnrichedContainersToWire } from '../lib/container-wire-mapper' @@ -177,17 +177,44 @@ function nextSublabel(containerType: ContainerSubtype | null, rawSublabel: strin return trimmed.length === 0 ? null : trimmed } +/** The tube barcode or Paper sublabel the Container has now. */ +async function currentIdentifier(database: Database, id: number, containerType: ContainerSubtype | null): Promise { + switch (containerType) { + case 'micronix_tube': + return (await database.select({ value: micronixTube.barcode }).from(micronixTube).where(eq(micronixTube.id, id)).get())?.value ?? null + case 'cryovial_tube': + return (await database.select({ value: cryovialTube.barcode }).from(cryovialTube).where(eq(cryovialTube.id, id)).get())?.value ?? null + case 'paper': + return (await database.select({ value: paper.sublabel }).from(paper).where(eq(paper.id, id)).get())?.value ?? null + default: + return null + } +} + +/** + * Rejects a barcode or sublabel another Container already has. A tube barcode is compared + * with every tube's barcode ignoring case and surrounding whitespace. An unchanged value is + * not checked, so a Container whose barcode was duplicated before this check existed still saves. + */ async function assertIdentifierFree( database: Database, id: number, + containerType: ContainerSubtype | null, identifier: string | null, label: 'Barcode' | 'Sublabel', ): Promise { if (identifier === null) return - const takenBy = await resolveContainerByBarcode(database, identifier) - if (takenBy !== null && takenBy !== id) { - throw new ValidationError(`${label} '${identifier}' is already in use`) - } + if (identifier === (await currentIdentifier(database, id, containerType))?.trim()) return + const takenBy = + label === 'Barcode' + ? await findTubesUsingBarcode(database, identifier, id) + : (await findContainersByBarcode(database, identifier)).filter((match) => match.containerId !== id) + if (takenBy.length === 0) return + // Name the stored spelling when only a case or whitespace variant collides. + const asVariant = takenBy.some((match) => match.barcode === identifier) ? '' : ` as '${takenBy[0].barcode}'` + throw new ValidationError(`${label} '${identifier}' is already in use${asVariant}`, { + containerIds: takenBy.map((match) => match.containerId), + }) } /** De-duplicated Tag ids, after checking that each one exists. */ @@ -231,9 +258,9 @@ containers.patch('/:id', memberMiddleware, async (c) => { } } const newBarcode = rawBarcode === undefined ? undefined : nextBarcode(containerType, rawBarcode) - if (newBarcode !== undefined) await assertIdentifierFree(tx, id, newBarcode, 'Barcode') + if (newBarcode !== undefined) await assertIdentifierFree(tx, id, containerType, newBarcode, 'Barcode') const newSublabel = rawSublabel === undefined ? undefined : nextSublabel(containerType, rawSublabel) - if (newSublabel !== undefined) await assertIdentifierFree(tx, id, newSublabel, 'Sublabel') + if (newSublabel !== undefined) await assertIdentifierFree(tx, id, containerType, newSublabel, 'Sublabel') const newTagIds = tagIds === undefined ? undefined : await knownTagIds(tx, tagIds) const [row] = await tx diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index a2ea1f23..a1bb846f 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -27,7 +27,7 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Study subjects with missing study.** Study subjects that reference a deleted study. - **Derivation broken references.** container_derivation rows with missing parent or child container. - **Storage container tag orphans.** Tag links that reference a deleted container or tag. -- **Duplicate barcodes (micronix).** Micronix tubes that share the same barcode (schema expects unique barcodes). Duplicate barcodes on cryovial tubes are **not** treated as a data integrity issue. +- **Duplicate tube barcodes.** Micronix and cryovial tubes that share a barcode, comparing without case or surrounding spaces, so `CK3-3FR4` and `ck3-3fr4` count as one barcode. **Spellings** lists each stored form when they differ. Moves and derivations refuse to pick between these tubes, so identify them by container id or position. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. - **Containers with no grid position.** Tubes or wells in a collection with no well or slot. Legacy rows are allowed and do **not** count toward the overview issue total. Assign a position via edit or scan move when the well is known. diff --git a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md index 8f366eeb..f399ce11 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/container-movement.md @@ -99,6 +99,7 @@ After execution, micronix and cryovial results are per file. Paper shows sheet c ## Errors +- Ambiguous barcode on Resolve: the barcode is on more than one container, and the error lists them. The move won't pick one. Fix the barcode, or move that tube some other way. - Unresolved micronix barcodes: they are listed on Resolve. Click **Back**, fix the CSV, then **Next: Resolve Containers**. You do not need to recreate destination plates that already exist. - Relocation error on Upload (tube at an empty well not relocated): every tube currently on the destination plate must stay in place (barcode in that well) or appear at another well in the same batch. - Occupied destination at execute: adjust positions or use a different atomicity mode. diff --git a/packages/docs/src/content/docs/guides/features/derivations.md b/packages/docs/src/content/docs/guides/features/derivations.md index ef4185a8..4a596fab 100644 --- a/packages/docs/src/content/docs/guides/features/derivations.md +++ b/packages/docs/src/content/docs/guides/features/derivations.md @@ -58,4 +58,6 @@ A repository example for DBS-to-DNA from controls is `examples/derivation-contro Type, date, protocol, and notes are read-only on the parent and child container pages. The app has no edit or delete for a derivation. Ask an administrator if a record is wrong. +If `parent_container_barcode` is on more than one container, the row fails with an ambiguous barcode error that lists them. Use `parent_container_id` for that row instead. + If the parent is not found, check identifier, source type (control vs subject), and parent container type. Position errors: use `A01` for micronix and static wells. diff --git a/packages/docs/src/content/docs/guides/troubleshooting/common-issues.md b/packages/docs/src/content/docs/guides/troubleshooting/common-issues.md index b7cfe8c2..d621712c 100644 --- a/packages/docs/src/content/docs/guides/troubleshooting/common-issues.md +++ b/packages/docs/src/content/docs/guides/troubleshooting/common-issues.md @@ -23,7 +23,9 @@ For more column rules, see [CSV file guidelines](/docs/guides/troubleshooting/cs **Not found.** Check barcode or position, collection, and that the container was not deleted. Identifiers must match exactly. -**Barcode already exists.** Micronix barcodes are unique across the system. Use a different barcode or open the existing container. +**Barcode already exists.** A new or changed micronix or cryovial barcode can't match any other tube's barcode, of either type, ignoring case and surrounding spaces. Use a different barcode or open the existing container. A container whose barcode was already shared before this rule still saves when you leave its barcode unchanged. + +**Ambiguous barcode.** The barcode is on more than one container, so the app won't guess which one you mean. The message lists the matching containers. Identify the container by its id or by its collection and position instead. **Data integrity** lists every shared tube barcode. **Position occupied or out of range.** `A13` is invalid on a 96-well plate (columns 01-12). Pick an empty well. diff --git a/packages/web/src/lib/api/admin.ts b/packages/web/src/lib/api/admin.ts index 19b57db1..e5bc021c 100644 --- a/packages/web/src/lib/api/admin.ts +++ b/packages/web/src/lib/api/admin.ts @@ -68,9 +68,12 @@ export const adminApi = { getIntegrityReport: () => api.get('/admin/data-audit/integrity-report'), } +/** Tubes whose barcodes are equal ignoring case and surrounding whitespace. */ export interface DuplicateBarcodeItem { barcode: string - containerType: 'micronix_tube' + /** Every stored spelling in the group; more than one means case or whitespace variants. */ + variants: string[] + containerTypes: Array<'micronix_tube' | 'cryovial_tube'> ids: number[] } diff --git a/packages/web/src/lib/api/collections.ts b/packages/web/src/lib/api/collections.ts index a8482df0..0daa9ba8 100644 --- a/packages/web/src/lib/api/collections.ts +++ b/packages/web/src/lib/api/collections.ts @@ -167,7 +167,15 @@ export const collectionsApi = { | { type: 'position'; sourceCollectionName: string; sourcePosition: string } > }) => - api.post<{ containers: Array<{ identifier: { type: string; value: string } | string; container: unknown }> }>('/collections/containers/resolve', data), + api.post<{ + containers: Array<{ + identifier: { type: string; value: string } | string + container: unknown + /** Set, with `container` null, when a barcode matches more than one Container. */ + error?: string + candidates?: Array<{ containerId: number; containerType: string; barcode: string }> + }> + }>('/collections/containers/resolve', data), listCollectionsByType: (type: 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' | 'sheet') => api.get<{ collections: Array<{ id: number; name: string; locationId?: number | null; itemCount?: number; location?: { id: number; path: string | null } | null }> }>(`/collections/list/${type}`), listAllCollections: () => diff --git a/packages/web/src/lib/scan-move/__tests__/resolve.test.ts b/packages/web/src/lib/scan-move/__tests__/resolve.test.ts index 640f1c44..36687f5a 100644 --- a/packages/web/src/lib/scan-move/__tests__/resolve.test.ts +++ b/packages/web/src/lib/scan-move/__tests__/resolve.test.ts @@ -78,6 +78,22 @@ describe('groupResolveResults', () => { expect(outcomes.flatMap((o) => o.errors)).toEqual([]) }) + it('reports an ambiguous barcode as an error on its row, not as a tube that could not be found', () => { + const error = "Ambiguous barcode 'BC2': it matches 2 containers (container 7 (micronix_tube in P1 A01), container 9 (cryovial_tube in B1 A01))." + const outcomes = groupResolveResults( + entries, + [ + { identifier: 'BC1', container: containerInfo({ containerId: 1 }) }, + { identifier: 'BC2', container: null, error }, + ], + files, + micronixScanMoveVariant, + ) + expect(outcomes[0].errors).toEqual([{ row: 2, error }]) + expect(outcomes[0].unresolvedContainers).toEqual([]) + expect(outcomes[0].resolvedContainers.map((r) => r.identifierKey)).toEqual(['BC1']) + }) + it('flags files whose containers are not in the variant collection type', () => { const outcomes = groupResolveResults( entries, diff --git a/packages/web/src/lib/scan-move/gateway.ts b/packages/web/src/lib/scan-move/gateway.ts index 0aa0f6d2..3e87e9e0 100644 --- a/packages/web/src/lib/scan-move/gateway.ts +++ b/packages/web/src/lib/scan-move/gateway.ts @@ -10,6 +10,7 @@ export function createCollectionsScanMoveGateway(variant: ScanMoveVariant): Scan containers: response.containers.map((entry) => ({ identifier: entry.identifier, container: (entry.container as ScanMoveContainerInfo | null | undefined) ?? null, + ...(entry.error !== undefined && { error: entry.error }), })), } }, diff --git a/packages/web/src/lib/scan-move/resolve.ts b/packages/web/src/lib/scan-move/resolve.ts index 2aacc504..3035a93e 100644 --- a/packages/web/src/lib/scan-move/resolve.ts +++ b/packages/web/src/lib/scan-move/resolve.ts @@ -60,11 +60,18 @@ export function groupResolveResults( const wrongTypeFileIndexes = new Set() for (const response of responseContainers) { - if (!response.container) continue const key = variant.identifierKeyFromResponse(response.identifier) if (key === null) continue const entry = entryByKey.get(key) if (!entry) continue + if (!response.container) { + // The server found several Containers for this identifier and won't pick one. + if (response.error !== undefined) { + resolvedKeys.add(key) + outcomes[entry.fileIndex].errors.push({ row: entry.rowIndex + 1, error: response.error }) + } + continue + } resolvedKeys.add(key) outcomes[entry.fileIndex].resolvedContainers.push({ identifierKey: key, diff --git a/packages/web/src/lib/scan-move/types.ts b/packages/web/src/lib/scan-move/types.ts index e364744b..aff7d2ef 100644 --- a/packages/web/src/lib/scan-move/types.ts +++ b/packages/web/src/lib/scan-move/types.ts @@ -176,6 +176,8 @@ export interface ScanMoveVariant { export interface ScanMoveResolveResponseEntry { identifier: unknown container: ScanMoveContainerInfo | null + /** Set when the identifier matched more than one Container (an ambiguous barcode). */ + error?: string } /** diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index d7e44eb2..946adda2 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -321,15 +321,16 @@ export default function AdminDataIntegrityReport() {
+ @@ -338,7 +339,10 @@ export default function AdminDataIntegrityReport() { {report.duplicateBarcodes.map((d, i) => ( - + + {reviewRows.map((row, rowIndex) => ( - + {reviewHeaders.map((header) => (
BarcodeSpellings Type Container IDs
{d.barcode}{d.containerType} + {d.variants.length > 1 ? d.variants.map((v) => JSON.stringify(v)).join(', ') : '—'} + {d.containerTypes.join(', ')} {d.ids.map((id) => ( diff --git a/packages/web/src/pages/__tests__/AdminDataIntegrityOverview.test.tsx b/packages/web/src/pages/__tests__/AdminDataIntegrityOverview.test.tsx index 23bca334..04c17e0e 100644 --- a/packages/web/src/pages/__tests__/AdminDataIntegrityOverview.test.tsx +++ b/packages/web/src/pages/__tests__/AdminDataIntegrityOverview.test.tsx @@ -46,7 +46,7 @@ describe('AdminDataIntegrityOverview', () => { it('does not count containers with no grid position as integrity issues', async () => { vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ ...emptyIntegrityReport(), - duplicateBarcodes: [{ barcode: 'DUP', containerType: 'micronix_tube', ids: [1, 2] }], + duplicateBarcodes: [{ barcode: 'DUP', variants: ['DUP'], containerTypes: ['micronix_tube'], ids: [1, 2] }], containersWithNoGridPosition: [ { id: 9, containerType: 'micronix_tube', collectionId: 3 }, ], diff --git a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx index c2baf923..005ec064 100644 --- a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx +++ b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import AdminDataIntegrityReport from '../AdminDataIntegrityReport' import { adminApi } from '../../lib/api/admin' @@ -56,6 +56,21 @@ describe('AdminDataIntegrityReport', () => { }) }) + it('lists duplicated tube barcodes with their spellings and tube types', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + duplicateBarcodes: [ + { barcode: 'CK3-3FR4', variants: ['CK3-3FR4', 'ck3-3fr4 '], containerTypes: ['cryovial_tube', 'micronix_tube'], ids: [42911, 137291] }, + ], + }) + await render() + const row = (await screen.findByRole('cell', { name: 'CK3-3FR4' })).closest('tr')! + expect(within(row).getByRole('cell', { name: '"CK3-3FR4", "ck3-3fr4 "' })).toBeInTheDocument() + expect(within(row).getByRole('cell', { name: 'cryovial_tube, micronix_tube' })).toBeInTheDocument() + expect(within(row).getByRole('link', { name: '137291' })).toHaveAttribute('href', '/containers/137291') + expect(screen.getByText('Duplicate tube barcodes')).toBeInTheDocument() + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() From 57d307971e480136e0583d41fff73363ea6634b5 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 18:58:46 -0700 Subject: [PATCH 21/54] fix(web): sort date columns with the display parser so server timestamps sort as UTC compareBySortType's 'date' key read strings with new Date(), which treats a zoneless SQLite timestamp as local time. It now goes through displayDateTime (#117), the one parser for server timestamps, so a 'date' column orders rows the same way it displays them (#116, #117). Co-Authored-By: Claude Opus 5.5 --- packages/web/src/lib/__tests__/sort-compare.test.ts | 6 ++++++ packages/web/src/lib/sort-compare.ts | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/web/src/lib/__tests__/sort-compare.test.ts b/packages/web/src/lib/__tests__/sort-compare.test.ts index a897e639..10ae9b8e 100644 --- a/packages/web/src/lib/__tests__/sort-compare.test.ts +++ b/packages/web/src/lib/__tests__/sort-compare.test.ts @@ -26,6 +26,12 @@ describe('compareBySortType', () => { expect(sorted([10, '9', 100, 0.5], 'number')).toEqual([0.5, '9', 10, 100]) }) + it('sorts a zoneless server timestamp as UTC', () => { + // Tests run in America/Los_Angeles: read as local time, 10:00 would be 18:00Z and sort last. + const dates = ['2024-01-15T12:00:00Z', '2024-01-15 10:00:00'] + expect(sorted(dates, 'date')).toEqual(['2024-01-15 10:00:00', '2024-01-15T12:00:00Z']) + }) + it('sorts dates chronologically', () => { const dates = ['2024-03-01', '2023-12-31T23:00:00Z', '2024-01-15'] expect(sorted(dates, 'date')).toEqual(['2023-12-31T23:00:00Z', '2024-01-15', '2024-03-01']) diff --git a/packages/web/src/lib/sort-compare.ts b/packages/web/src/lib/sort-compare.ts index 1ca3ac19..ae795c63 100644 --- a/packages/web/src/lib/sort-compare.ts +++ b/packages/web/src/lib/sort-compare.ts @@ -1,3 +1,5 @@ +import { displayDateTime } from './date-utils' + /** How a sortable value is ordered. */ export type SortType = 'text' | 'number' | 'date' @@ -28,7 +30,8 @@ function sortKey(value: unknown, type: SortType): string | number | null { if (value instanceof Date) return Number.isNaN(value.getTime()) ? null : value.getTime() if (typeof value !== 'string' && typeof value !== 'number') return null if (typeof value === 'string' && value.trim() === '') return null - const t = new Date(value).getTime() + // A string goes through the display parser, so a zoneless server timestamp sorts as UTC. + const t = typeof value === 'string' ? displayDateTime(value) : new Date(value).getTime() return Number.isNaN(t) ? null : t } } From e5371d1f2c95ee4d86abfa39f0e0e953b5064699 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:01:35 -0700 Subject: [PATCH 22/54] fix(api): reject malformed created_from/created_to on the studies and subjects lists #118 added created-date filters to GET /studies and GET /subjects and passed the raw query strings to dateRangeConditions. #104 validates these bounds on the specimens list with timestampFilter (400 VALIDATION_ERROR naming the parameter), because a malformed bound compared as a string matches everything or nothing. The two new lists now go through timestampFilter too. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/routes/__tests__/studies.test.ts | 11 +++++++++++ packages/api/src/routes/__tests__/subjects.test.ts | 11 +++++++++++ packages/api/src/routes/studies.ts | 5 +++-- packages/api/src/routes/subjects.ts | 7 ++++++- 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index 8b600254..236385fe 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -57,6 +57,17 @@ describe('Studies API (list, get, create, update)', () => { expect(((await toOnly.json()) as { pagination: { total: number } }).pagination.total).toBe(3) }) + it.each([ + ['created_from', 'yesterday'], + ['created_to', '2024-13-01'], + ])('returns 400 for %s=%s', async (param, value) => { + const res = await ctx.request(`/api/studies?${param}=${encodeURIComponent(value)}`) + + expect(res.status).toBe(400) + const data = (await res.json()) as { error: string } + expect(data.error).toContain(param) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/studies', { method: 'GET' }) expect(res.status).toBe(401) diff --git a/packages/api/src/routes/__tests__/subjects.test.ts b/packages/api/src/routes/__tests__/subjects.test.ts index 58ce7f3a..4c558a58 100644 --- a/packages/api/src/routes/__tests__/subjects.test.ts +++ b/packages/api/src/routes/__tests__/subjects.test.ts @@ -56,6 +56,17 @@ describe('Subjects API', () => { expect(((await fromOnly.json()) as { pagination: { total: number } }).pagination.total).toBe(3) }) + it.each([ + ['created_from', 'yesterday'], + ['created_to', '2024-13-01'], + ])('returns 400 for %s=%s', async (param, value) => { + const res = await ctx.request(`/api/subjects?${param}=${encodeURIComponent(value)}`) + + expect(res.status).toBe(400) + const data = (await res.json()) as { error: string } + expect(data.error).toContain(param) + }) + it('returns 401 when not authenticated', async () => { const res = await authenticatedRequest(ctx.createRequestApp(), '/api/subjects', { method: 'GET' }) expect(res.status).toBe(401) diff --git a/packages/api/src/routes/studies.ts b/packages/api/src/routes/studies.ts index 3159bf1e..7500e471 100644 --- a/packages/api/src/routes/studies.ts +++ b/packages/api/src/routes/studies.ts @@ -25,6 +25,7 @@ import { requireParam } from '../lib/common-validators' import { withWriteTransaction } from '../db/write-transaction' import { assertNotUsedInQpcr } from '../lib/qpcr-usage' import { insertedRow } from '../lib/inserted-row' +import { timestampFilter } from '../lib/date-filters' /** Short code prefix for tutorial namespace. Any study whose short code starts with this (case-insensitive) may be deleted by any member. Only admins may rename a study into or out of this namespace. */ const TUTORIAL_SHORT_CODE_PREFIX = 'TUT' @@ -63,8 +64,8 @@ studies.get('/', authMiddleware, async (c) => { return c.json( await listStudies(database, { search, - createdFrom: c.req.query('created_from'), - createdTo: c.req.query('created_to'), + createdFrom: timestampFilter('created_from', c.req.query('created_from')), + createdTo: timestampFilter('created_to', c.req.query('created_to')), page, limit, }), diff --git a/packages/api/src/routes/subjects.ts b/packages/api/src/routes/subjects.ts index 366ee264..6fb6d62a 100644 --- a/packages/api/src/routes/subjects.ts +++ b/packages/api/src/routes/subjects.ts @@ -12,6 +12,7 @@ import { import { eq, sql, and, inArray, or, isNull } from 'drizzle-orm' import { validatePage, validateLimit } from '../lib/constants' import { dateRangeConditions } from '../lib/sql-helpers' +import { timestampFilter } from '../lib/date-filters' import { z } from 'zod' import { resolveStudyByShortCode, resolveSubjectByNameAndStudy, resolveSpecimenTypeByName } from '../lib/identifier-resolution' import { validateSubjectName, validateStudyShortCode, validateSpecimenData, validateCollectionDate, validateContainerTypeForSpecimenType, validateUnitForContainerType } from '../lib/validation' @@ -50,7 +51,11 @@ subjects.get('/', authMiddleware, async (c) => { const limit = await validateLimit(dbInstance, c.req.query('limit')) const offset = (page - 1) * limit const whereClause = and( - ...dateRangeConditions(studySubject.created, c.req.query('created_from'), c.req.query('created_to')), + ...dateRangeConditions( + studySubject.created, + timestampFilter('created_from', c.req.query('created_from')), + timestampFilter('created_to', c.req.query('created_to')), + ), ) const [subjectsList, countResult] = await Promise.all([ From 3162282a8997cb9a3d53f017f58521fd681f67d8 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:39:57 -0700 Subject: [PATCH 23/54] fix(qpcr): report amplification rows skipped for wells with no result A QuantStudio export has an amplification curve for all 96 wells but a Results row only for wells set up with a sample. The upload stored only curves that match a well result and dropped the rest silently. In the example file those are 3,000 rows from 50 empty wells (no sample, no target name, Delta Rn 0 at every cycle), so they stay skipped, but the upload response now returns skippedAmplification { rowCount, wellPositions } and stored counts, and the experiment page says how many rows were skipped, from which wells, and why. Closes #129 Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/qpcr-results-write.ts | 31 +++++++++- .../routes/__tests__/qpcr-experiments.test.ts | 60 +++++++++++++++++++ packages/api/src/routes/qpcr-experiments.ts | 4 +- .../docs/guides/features/qpcr-experiments.md | 8 +++ .../qpcr-results-upload-message.test.ts | 28 +++++++++ packages/web/src/lib/api/qpcr.ts | 10 +++- .../src/lib/qpcr-results-upload-message.ts | 16 +++++ .../web/src/pages/QpcrExperimentDetail.tsx | 11 +++- 8 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 packages/web/src/lib/__tests__/qpcr-results-upload-message.test.ts create mode 100644 packages/web/src/lib/qpcr-results-upload-message.ts diff --git a/packages/api/src/lib/qpcr-results-write.ts b/packages/api/src/lib/qpcr-results-write.ts index 5d9c4536..f5b45f67 100644 --- a/packages/api/src/lib/qpcr-results-write.ts +++ b/packages/api/src/lib/qpcr-results-write.ts @@ -15,10 +15,24 @@ function wellTargetKey(wellPosition: string, targetName: string | null): string return `${wellPosition}\t${targetName ?? ''}` } +/** What an upload stored, and the amplification rows it left out. */ +export interface RecordedQpcrResults { + run: typeof qpcrRun.$inferSelect + wellResultCount: number + amplificationCount: number + /** + * Amplification rows whose well and target have no row in the results section. QuantStudio + * exports a curve for all 96 wells but a result only for wells set up with a sample, so these + * are empty wells; the upload leaves them out and reports them instead. + */ + skippedAmplification: { rowCount: number; wellPositions: string[] } +} + /** * Record a parsed qPCR result file against an experiment: the run, its well results, their * amplification data, and the experiment's `results_uploaded` status. All of it commits in one * transaction, so a failure partway leaves no run or well result behind and a retry starts clean. + * Amplification rows with no matching well result are not stored; the result counts them. */ export async function recordQpcrResults( database: Database, @@ -28,7 +42,7 @@ export async function recordQpcrResults( parseResult: QpcrParseResult userId: number | null }, -): Promise { +): Promise { const { experimentId, instrumentType, parseResult, userId } = input const now = utcNow() @@ -83,9 +97,15 @@ export async function recordQpcrResults( } const ampRows: Array = [] + let skippedRowCount = 0 + const skippedWells = new Set() for (const row of parseResult.amplificationData) { const wellResultId = wellResultIds.get(wellTargetKey(row.wellPosition, row.targetName)) - if (wellResultId == null) continue + if (wellResultId == null) { + skippedRowCount++ + skippedWells.add(row.wellPosition) + continue + } ampRows.push({ qpcrWellResultId: wellResultId, cycle: row.cycle, @@ -102,6 +122,11 @@ export async function recordQpcrResults( .set({ status: 'results_uploaded', lastUpdated: now, updatedBy: userId }) .where(eq(qpcrExperiment.id, experimentId)) - return runRow + return { + run: runRow, + wellResultCount: wellResultRows.length, + amplificationCount: ampRows.length, + skippedAmplification: { rowCount: skippedRowCount, wellPositions: [...skippedWells] }, + } }) } diff --git a/packages/api/src/routes/__tests__/qpcr-experiments.test.ts b/packages/api/src/routes/__tests__/qpcr-experiments.test.ts index 63db3147..fb7f50be 100644 --- a/packages/api/src/routes/__tests__/qpcr-experiments.test.ts +++ b/packages/api/src/routes/__tests__/qpcr-experiments.test.ts @@ -145,6 +145,66 @@ describe('qPCR Experiments Template', () => { expect(amp).toHaveLength(2) }) + it('skips and reports amplification rows for a well with no result row', async () => { + // As in a QuantStudio export: A1 has a sample and a result; A2 is an empty well, so it has + // a flat curve with no target name in Amplification Data and no row in Results. + const XLSX = await import('xlsx') + const wb = XLSX.utils.book_new() + XLSX.utils.book_append_sheet( + wb, + XLSX.utils.aoa_to_sheet([ + ['Well', 'Well Position', 'Sample Name', 'Target Name', 'Task', 'CT'], + [1, 'A1', 'S1', 'Target 1', 'UNKNOWN', 22.1], + ]), + 'Results', + ) + XLSX.utils.book_append_sheet( + wb, + XLSX.utils.aoa_to_sheet([ + ['Well', 'Well Position', 'Cycle', 'Target Name', 'Rn', 'Delta Rn'], + [1, 'A1', 1, 'Target 1', 0.5, 0.01], + [1, 'A1', 2, 'Target 1', 0.6, 0.02], + [2, 'A2', 1, '', 0.4, 0], + [2, 'A2', 2, '', 0.4, 0], + [2, 'A2', 3, '', 0.4, 0], + ]), + 'Amplification Data', + ) + const fileContent = (XLSX.write(wb, { type: 'buffer', bookType: 'xls' }) as Buffer).toString('base64') + const id = await createExperiment() + + const res = await ctx.request(`/api/qpcr-experiments/${id}/results`, { + method: 'POST', + json: { fileContent, fileName: 'run.xls', instrumentType: 'QuantStudio' }, + }) + + expect(res.status).toBe(201) + const body = (await res.json()) as { + wellResultCount: number + amplificationCount: number + skippedAmplification: { rowCount: number; wellPositions: string[] } + } + expect(body.wellResultCount).toBe(1) + expect(body.amplificationCount).toBe(2) + expect(body.skippedAmplification).toEqual({ rowCount: 3, wellPositions: ['A02'] }) + expect(await countRows()).toEqual({ runs: 1, wellResults: 1, amplification: 2 }) + }) + + it('reports no skipped amplification rows for a Bio-Rad file, which has none', async () => { + const csv = ['Well,Fluor,Target,Content,Sample,Cq,Starting Quantity (SQ),Cq Mean', 'A01,FAM,,Unkn,S1,24.6,10140,24.6'].join('\n') + const id = await createExperiment() + + const res = await ctx.request(`/api/qpcr-experiments/${id}/results`, { + method: 'POST', + json: { fileContent: Buffer.from(csv).toString('base64'), fileName: 'run.csv', instrumentType: 'Biorad_CFX' }, + }) + + expect(res.status).toBe(201) + const body = (await res.json()) as { amplificationCount: number; skippedAmplification: { rowCount: number } } + expect(body.amplificationCount).toBe(0) + expect(body.skippedAmplification.rowCount).toBe(0) + }) + async function createExperiment(): Promise { const res = await ctx.request('/api/qpcr-experiments', { method: 'POST', diff --git a/packages/api/src/routes/qpcr-experiments.ts b/packages/api/src/routes/qpcr-experiments.ts index 1f9ff14f..a5340bda 100644 --- a/packages/api/src/routes/qpcr-experiments.ts +++ b/packages/api/src/routes/qpcr-experiments.ts @@ -643,14 +643,14 @@ export function createQpcrExperimentsRoutes(database: Database): Hono { parseResult = await parseQuantStudioXls(buffer, data.fileName) } - const runRow = await recordQpcrResults(database, { + const recorded = await recordQpcrResults(database, { experimentId: id, instrumentType: data.instrumentType, parseResult, userId: (c.get('user') as { id: number } | undefined)?.id ?? null, }) - return c.json({ run: runRow, wellResultCount: parseResult.wellResults.length, amplificationCount: parseResult.amplificationData.length }, 201) + return c.json(recorded, 201) } catch (error) { if (error instanceof z.ZodError) { return c.json({ error: 'Validation error', details: error.issues, errorCode: 'VALIDATION_ERROR' }, 400) diff --git a/packages/docs/src/content/docs/guides/features/qpcr-experiments.md b/packages/docs/src/content/docs/guides/features/qpcr-experiments.md index 01172f0c..25d849bb 100644 --- a/packages/docs/src/content/docs/guides/features/qpcr-experiments.md +++ b/packages/docs/src/content/docs/guides/features/qpcr-experiments.md @@ -43,6 +43,14 @@ Load the template on the instrument. That step is not in SampleDB. Disabled. When it returns: upload Bio-Rad CSV or Quant Studio XLS, matching the instrument type. Amplification data is stored when present. +### Amplification data from empty wells + +A Quant Studio export has an amplification curve for all 96 wells, but a results row only for wells that were set up with a sample. The import stores a curve only when its well and target have a results row, so curves from empty wells are skipped. After the upload, the page reports how many amplification rows were skipped and from which wells. + +For example, the Quant Studio file for run varATS-IM-25-048 has 5,760 amplification rows (96 wells × 60 cycles). The import stores the 2,760 rows for the 46 wells with a result and skips 3,000 rows from 50 wells (A7–F12, G6–H12). Those wells have no sample name on the Sample Setup sheet, no row on the Results sheet, and no target name on the Amplification Data sheet, and their Delta Rn is 0 at every cycle. They hold no sample, standard, or control, so their curves carry no information. + +Bio-Rad CSV exports have no amplification data, so nothing is skipped. + ## Experiment list Columns: **Name**, **Status** (Setup, In progress, Results imported), **Template**, **Plate**, **Target(s)**, **Assay** (fluorophore/reporter, or Multiple for multiplex), **Wells**, **Runs**, **Last run**, **Created**, **Updated**. Filter by status. Default sort is most recently updated. Click a row to open it. diff --git a/packages/web/src/lib/__tests__/qpcr-results-upload-message.test.ts b/packages/web/src/lib/__tests__/qpcr-results-upload-message.test.ts new file mode 100644 index 00000000..542893ee --- /dev/null +++ b/packages/web/src/lib/__tests__/qpcr-results-upload-message.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect } from 'vitest' +import { qpcrResultsUploadMessage } from '../qpcr-results-upload-message' + +describe('qpcrResultsUploadMessage', () => { + it('reports what was stored when nothing was skipped', () => { + expect( + qpcrResultsUploadMessage({ + run: { id: 1 }, + wellResultCount: 96, + amplificationCount: 0, + skippedAmplification: { rowCount: 0, wellPositions: [] }, + }), + ).toBe('Imported 96 well results and 0 amplification rows.') + }) + + it('reports skipped amplification rows, why, and which wells', () => { + expect( + qpcrResultsUploadMessage({ + run: { id: 1 }, + wellResultCount: 46, + amplificationCount: 2760, + skippedAmplification: { rowCount: 3000, wellPositions: ['G06', 'H06'] }, + }), + ).toBe( + 'Imported 46 well results and 2,760 amplification rows. Skipped 3,000 amplification rows from 2 wells with no result in the file (empty wells): G06, H06.', + ) + }) +}) diff --git a/packages/web/src/lib/api/qpcr.ts b/packages/web/src/lib/api/qpcr.ts index 244abe89..1202c4c2 100644 --- a/packages/web/src/lib/api/qpcr.ts +++ b/packages/web/src/lib/api/qpcr.ts @@ -1,4 +1,12 @@ import { api } from './client' + +/** What a result upload stored, and the amplification rows it skipped because their well has no result row. */ +export interface QpcrResultsUploadResponse { + run: { id: number } + wellResultCount: number + amplificationCount: number + skippedAmplification: { rowCount: number; wellPositions: string[] } +} // qPCR experiments export interface QpcrExperimentTarget { id: number @@ -70,7 +78,7 @@ export const qpcrExperimentsApi = { updateWells: (id: number, data: { wellPosition?: string; positions?: string[]; contentType: 'empty' | 'negative' }) => api.patch<{ wells: QpcrExperimentWell[] }>(`/qpcr-experiments/${id}/wells`, data), uploadResults: (id: number, data: { fileContent: string; fileName: string; instrumentType: 'Biorad_CFX' | 'QuantStudio' }) => - api.post<{ run: { id: number }; wellResultCount: number; amplificationCount: number }>(`/qpcr-experiments/${id}/results`, data), + api.post(`/qpcr-experiments/${id}/results`, data), delete: (id: number) => api.delete(`/qpcr-experiments/${id}`), } diff --git a/packages/web/src/lib/qpcr-results-upload-message.ts b/packages/web/src/lib/qpcr-results-upload-message.ts new file mode 100644 index 00000000..3f477ec5 --- /dev/null +++ b/packages/web/src/lib/qpcr-results-upload-message.ts @@ -0,0 +1,16 @@ +/** + * User-facing summary of a qPCR result upload: what was stored, and the amplification rows the + * upload skipped because their well has no row in the file's results section (empty wells). + */ +import type { QpcrResultsUploadResponse } from './api/qpcr' + +function plural(n: number, one: string, many: string): string { + return `${n.toLocaleString('en-US')} ${n === 1 ? one : many}` +} + +export function qpcrResultsUploadMessage(result: QpcrResultsUploadResponse): string { + const stored = `Imported ${plural(result.wellResultCount, 'well result', 'well results')} and ${plural(result.amplificationCount, 'amplification row', 'amplification rows')}.` + const { rowCount, wellPositions } = result.skippedAmplification + if (rowCount === 0) return stored + return `${stored} Skipped ${plural(rowCount, 'amplification row', 'amplification rows')} from ${plural(wellPositions.length, 'well', 'wells')} with no result in the file (empty wells): ${wellPositions.join(', ')}.` +} diff --git a/packages/web/src/pages/QpcrExperimentDetail.tsx b/packages/web/src/pages/QpcrExperimentDetail.tsx index 2133f025..82a7ee9e 100644 --- a/packages/web/src/pages/QpcrExperimentDetail.tsx +++ b/packages/web/src/pages/QpcrExperimentDetail.tsx @@ -3,6 +3,7 @@ import { useParams, Link, useNavigate, useSearchParams } from 'react-router-dom' import { useQueryClient } from '@tanstack/react-query' import { api } from '../lib/api/client' import { containersApi, type ContainerDetail } from '../lib/api/containers' +import { qpcrResultsUploadMessage } from '../lib/qpcr-results-upload-message' import { qpcrExperimentsApi } from '../lib/api/qpcr' import type { QpcrExperimentDetailResponse, QpcrExperimentWell } from '../lib/api/qpcr' import EntityBreadcrumbs from '../components/EntityBreadcrumbs' @@ -147,6 +148,7 @@ export default function QpcrExperimentDetail() { const [plateError, setPlateError] = useState(null) const [resultsUploading, setResultsUploading] = useState(false) const [resultsError, setResultsError] = useState(null) + const [resultsMessage, setResultsMessage] = useState(null) const [templateDownloading, setTemplateDownloading] = useState<'biorad' | 'quant_studio' | null>(null) const instrumentSelectRef = useRef(null) const [targets, setTargets] = useState(() => [{ key: '1', targetName: DEFAULT_TARGET_NAME, fluorophore: DEFAULT_FLUOROPHORE, reporter: DEFAULT_REPORTER }]) @@ -951,6 +953,11 @@ export default function QpcrExperimentDetail() { {resultsError}

)} + {resultsMessage && ( +

+ {resultsMessage} +

+ )}
@@ -972,12 +979,14 @@ export default function QpcrExperimentDetail() { const file = e.target.files?.[0] if (!file || !id) return setResultsError(null) + setResultsMessage(null) setResultsUploading(true) try { const buf = await file.arrayBuffer() const base64 = bytesToBase64(new Uint8Array(buf)) const resultInstrument = (instrumentSelectRef.current?.value as 'Biorad_CFX' | 'QuantStudio' | undefined) ?? 'Biorad_CFX' - await qpcrExperimentsApi.uploadResults(parseInt(id), { fileContent: base64, fileName: file.name, instrumentType: resultInstrument }) + const uploaded = await qpcrExperimentsApi.uploadResults(parseInt(id), { fileContent: base64, fileName: file.name, instrumentType: resultInstrument }) + setResultsMessage(qpcrResultsUploadMessage(uploaded)) invalidateQpcrExperimentQueries(queryClient) loadData() } catch (err: unknown) { From e6d1bc69da80ccb95fc47b731fd57d91c1e1d809 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:42:05 -0700 Subject: [PATCH 24/54] fix: Studies search matches lead person and description (#128) The Studies page placeholder and user guide say search matches title, code, lead person and description, but GET /api/studies only matched title and short code, so searching a lead such as "Greenhouse" returned nothing. listStudies now matches all four fields with likeContains. The page sent the search to the server and then filtered the results again in the browser. It now shows what the server returns, so the API is the single definition of a match. The guide lists the four fields. Co-Authored-By: Claude Opus 5.5 --- .../api/src/__tests__/helpers/factories.ts | 2 +- .../lib/studies/__tests__/study-read.test.ts | 24 +++++++ packages/api/src/lib/studies/study-read.ts | 14 +++- .../api/src/routes/__tests__/studies.test.ts | 19 ++++++ .../content/docs/guides/workflows/studies.md | 2 +- packages/web/src/pages/Studies.tsx | 16 +---- .../web/src/pages/__tests__/Studies.test.tsx | 67 +++++++++++++++++++ 7 files changed, 127 insertions(+), 17 deletions(-) diff --git a/packages/api/src/__tests__/helpers/factories.ts b/packages/api/src/__tests__/helpers/factories.ts index abe595c7..d8614505 100644 --- a/packages/api/src/__tests__/helpers/factories.ts +++ b/packages/api/src/__tests__/helpers/factories.ts @@ -356,7 +356,7 @@ export async function createTestControlDefinition(db: Database, data?: { */ export async function createTestStudy( db: Database, - data: { title: string, shortCode: string, leadPerson?: string, created?: string }, + data: { title: string, shortCode: string, leadPerson?: string, description?: string, created?: string }, ) { const now = utcNow() const [result] = await db.insert(study).values({ diff --git a/packages/api/src/lib/studies/__tests__/study-read.test.ts b/packages/api/src/lib/studies/__tests__/study-read.test.ts index 3d41c275..60454642 100644 --- a/packages/api/src/lib/studies/__tests__/study-read.test.ts +++ b/packages/api/src/lib/studies/__tests__/study-read.test.ts @@ -54,6 +54,30 @@ describe('study-read', () => { expect(result.studies.some((s) => s.shortCode === 'OTH01')).toBe(false) }) + describe('search matches each field', () => { + beforeEach(async () => { + await createTestStudy(testDb, { title: 'Malaria cohort', shortCode: 'MAL01', leadPerson: 'Ada Lovelace' }) + await createTestStudy(testDb, { + title: 'Vector survey', + shortCode: 'VEC02', + leadPerson: 'Greenhouse lab', + description: 'Household mosquito collections in Tororo', + }) + await createTestStudy(testDb, { title: 'Serology', shortCode: 'SER03', leadPerson: 'Greenhouse Lab' }) + }) + + it.each([ + ['title', 'malaria', ['MAL01']], + ['short code', 'vec0', ['VEC02']], + ['lead person, in any letter case', 'greenhouse', ['VEC02', 'SER03']], + ['description', 'mosquito collections', ['VEC02']], + ])('matches %s', async (_field, search, codes) => { + const result = await listStudies(testDb, { search, page: 1, limit: 10 }) + expect(result.studies.map((s) => s.shortCode).sort()).toEqual([...codes].sort()) + expect(result.pagination.total).toBe(codes.length) + }) + }) + it('matches "_" and "%" in the search term literally', async () => { await createTestStudy(testDb, { title: 'Cohort A_1', shortCode: 'UND01' }) await createTestStudy(testDb, { title: 'Cohort AB1', shortCode: 'WLD01' }) diff --git a/packages/api/src/lib/studies/study-read.ts b/packages/api/src/lib/studies/study-read.ts index 19dd1598..b40cd6fc 100644 --- a/packages/api/src/lib/studies/study-read.ts +++ b/packages/api/src/lib/studies/study-read.ts @@ -37,13 +37,23 @@ export type GetStudySummariesResult = { summaries: StudySummary[] } -/** List studies with optional search and pagination. */ +/** + * List studies with optional search and pagination. Search matches title, short code, + * lead person, or description, case-insensitively for ASCII letters. + */ export async function listStudies(database: Database, params: ListStudiesParams): Promise { const { search, createdFrom, createdTo, page, limit } = params const offset = (page - 1) * limit const whereClause = and( - search ? or(likeContains(study.title, search), likeContains(study.shortCode, search)) : undefined, + search + ? or( + likeContains(study.title, search), + likeContains(study.shortCode, search), + likeContains(study.leadPerson, search), + likeContains(study.description, search), + ) + : undefined, ...dateRangeConditions(study.created, createdFrom, createdTo), ) diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index 236385fe..8dbf8135 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -42,6 +42,25 @@ describe('Studies API (list, get, create, update)', () => { expect(data.pagination).toHaveProperty('totalPages') }) + it.each([ + ['lead person', 'Greenhouse'], + ['description', 'mosquito'], + ])('returns the study whose %s matches search', async (_field, search) => { + await createTestStudy(ctx.db, { + title: 'Vector survey', + shortCode: 'VEC02', + leadPerson: 'Greenhouse lab', + description: 'Household mosquito collections', + }) + await createTestStudy(ctx.db, { title: 'Serology', shortCode: 'SER03', leadPerson: 'Ada Lovelace' }) + + const res = await ctx.request(`/api/studies?search=${encodeURIComponent(search)}`) + expect(res.status).toBe(200) + const data = (await res.json()) as { studies: { shortCode: string }[]; pagination: { total: number } } + expect(data.studies.map((s) => s.shortCode)).toEqual(['VEC02']) + expect(data.pagination.total).toBe(1) + }) + it('counts only studies created inside created_from and created_to', async () => { await createTestStudy(ctx.db, { title: 'Before', shortCode: 'B1', created: '2026-08-31 23:59:59' }) await createTestStudy(ctx.db, { title: 'First day', shortCode: 'F1', created: '2026-09-01 00:00:00' }) diff --git a/packages/docs/src/content/docs/guides/workflows/studies.md b/packages/docs/src/content/docs/guides/workflows/studies.md index 50ebff82..ec711cfe 100644 --- a/packages/docs/src/content/docs/guides/workflows/studies.md +++ b/packages/docs/src/content/docs/guides/workflows/studies.md @@ -52,7 +52,7 @@ Use the same short code in CSV files that you use in the UI. Avoid special chara ## Find studies -On the Studies page, search matches titles, short codes, and lead person names. You can filter by type and lead, and sort the whole list by last updated date, title, lead, or subject, specimen, and container counts. Text sorts read numbers in order, so `2` comes before `10`. +On the Studies page, search matches titles, short codes, lead person names, and descriptions, ignoring letter case. You can filter by type and lead, and sort the whole list by last updated date, title, lead, or subject, specimen, and container counts. Text sorts read numbers in order, so `2` comes before `10`. ## Add subjects diff --git a/packages/web/src/pages/Studies.tsx b/packages/web/src/pages/Studies.tsx index 67cb265a..4991410e 100644 --- a/packages/web/src/pages/Studies.tsx +++ b/packages/web/src/pages/Studies.tsx @@ -109,7 +109,8 @@ export default function Studies() { ) // Load every study rather than pages of 50, so sorting and filtering cover the whole - // list. The search query narrows by title and short code on the server. + // list. Search runs on the server, which matches title, short code, lead person, and + // description; the page shows what it returns and does not filter the text again. const allStudiesQuery = useAllStudies('') const searchQuery = useAllStudies(debouncedSearch, { enabled: debouncedSearch !== '' }) const leadPersonsQuery = useStudyLeadPersons() @@ -183,17 +184,6 @@ export default function Studies() { const filteredAndSortedStudies = useMemo(() => { let filtered = [...studies] - if (debouncedSearch) { - const searchLower = debouncedSearch.toLowerCase() - filtered = filtered.filter( - (study) => - study.title.toLowerCase().includes(searchLower) || - study.shortCode.toLowerCase().includes(searchLower) || - study.leadPerson.toLowerCase().includes(searchLower) || - study.description?.toLowerCase().includes(searchLower) - ) - } - if (filterType !== 'all') { filtered = filtered.filter((study) => filterType === 'longitudinal' ? study.isLongitudinal : !study.isLongitudinal @@ -210,7 +200,7 @@ export default function Studies() { }) return filtered - }, [studies, debouncedSearch, filterType, filterLead, sortBy, sortDirection]) + }, [studies, filterType, filterLead, sortBy, sortDirection]) const displayStudies = filteredAndSortedStudies const filteredTotal = filteredAndSortedStudies.length diff --git a/packages/web/src/pages/__tests__/Studies.test.tsx b/packages/web/src/pages/__tests__/Studies.test.tsx index 4f98da1b..84cfe01f 100644 --- a/packages/web/src/pages/__tests__/Studies.test.tsx +++ b/packages/web/src/pages/__tests__/Studies.test.tsx @@ -88,6 +88,73 @@ describe('Studies', () => { await waitFor(() => expect(typeFilter).toHaveValue('all')) }) + describe('search', () => { + const studies: Study[] = [ + { id: 1, title: 'Malaria cohort', shortCode: 'MAL01', isLongitudinal: false, leadPerson: 'Ada Lovelace', created: '2020-01-01T00:00:00Z', lastUpdated: '2024-01-01T00:00:00Z' }, + { id: 2, title: 'Vector survey', shortCode: 'VEC02', isLongitudinal: false, leadPerson: 'Greenhouse lab', description: 'Household mosquito collections', created: '2020-01-01T00:00:00Z', lastUpdated: '2024-01-01T00:00:00Z' }, + { id: 3, title: 'Serology', shortCode: 'SER03', isLongitudinal: false, leadPerson: 'Greenhouse Lab', created: '2020-01-01T00:00:00Z', lastUpdated: '2024-01-01T00:00:00Z' }, + ] + + // Stands in for the server, which matches title, short code, lead person, and description. + function serverSearch(search: string | undefined): Study[] { + if (!search) return studies + const text = search.toLowerCase() + return studies.filter((study) => + [study.title, study.shortCode, study.leadPerson, study.description ?? ''].some((field) => field.toLowerCase().includes(text)) + ) + } + + beforeEach(() => { + vi.mocked(studiesApi.list).mockImplementation((search) => { + const list = serverSearch(search) + return Promise.resolve({ studies: list, pagination: { page: 1, limit: 1000, total: list.length, totalPages: 1 } }) + }) + }) + + it('says which fields search matches', async () => { + const { default: Studies } = await import('../Studies') + await render() + + expect(await screen.findByRole('textbox', { name: 'Search studies' })).toHaveAttribute( + 'placeholder', + 'Search by title, code, lead person, or description...' + ) + }) + + it.each([ + ['title', 'malaria', ['Malaria cohort']], + ['short code', 'VEC0', ['Vector survey']], + ['lead person', 'Greenhouse', ['Serology', 'Vector survey']], + ['description', 'mosquito', ['Vector survey']], + ])('shows the studies whose %s matches', async (_field, search, titles) => { + const user = userEvent.setup() + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?sort=title'] }) + + await user.type(await screen.findByRole('textbox', { name: 'Search studies' }), search) + + await waitFor(() => expect(studiesApi.list).toHaveBeenCalledWith(search, expect.anything())) + await waitFor(() => + expect(screen.getAllByRole('heading', { level: 3 }).map((h) => h.textContent)).toEqual(titles) + ) + }) + + it('leaves matching to the server and does not filter its results again', async () => { + // A server match the page could not reproduce from the fields it holds. + vi.mocked(studiesApi.list).mockImplementation((search) => { + const list = search ? [studies[0]] : studies + return Promise.resolve({ studies: list, pagination: { page: 1, limit: 1000, total: list.length, totalPages: 1 } }) + }) + const { default: Studies } = await import('../Studies') + await render(, { initialEntries: ['/studies?q=Tororo'] }) + + await waitFor(() => + expect(screen.getAllByRole('heading', { level: 3 }).map((h) => h.textContent)).toEqual(['Malaria cohort']) + ) + expect(studiesApi.list).toHaveBeenCalledWith('Tororo', expect.anything()) + }) + }) + describe('sorting', () => { function study(id: number, title: string, leadPerson: string, lastUpdated: string): Study { return { From 21d7dbfe1b04b83208ac39efd469c3365a270a25 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:43:31 -0700 Subject: [PATCH 25/54] fix(web): name barcode-only Derivations import Collections after the barcode (#130) The Derivations bulk import named a plate or cryovial box given only by collection_barcode "Collection-", while bulk import and the server's registration planner name it after the barcode. A later bulk import that gave that barcode as plate_name or box_name created a second Collection. newCollectionName in @sampledb/contract now holds the rule, and the registration planner, the bulk import wizard, and the Derivations import all use it. The Derivations import no longer falls back to a timestamped name; a missing Collection with neither a name nor a barcode is reported as an error instead of created. Co-Authored-By: Claude Opus 5.5 --- .../src/lib/__tests__/derivations-csv.test.ts | 117 +++++++++++++++++- .../lib/specimen-registration/collections.ts | 4 +- .../src/__tests__/collection-name.test.ts | 19 +++ packages/contract/src/collection-name.ts | 18 +++ packages/contract/src/index.ts | 1 + .../docs/guides/features/derivations.md | 2 + .../web/src/components/BulkImportFlow.tsx | 7 +- .../__tests__/effects.test.ts | 60 +++++++++ .../lib/derivations-bulk-import/effects.ts | 18 ++- .../src/lib/derivations-bulk-import/index.ts | 1 - 10 files changed, 228 insertions(+), 19 deletions(-) create mode 100644 packages/contract/src/__tests__/collection-name.test.ts create mode 100644 packages/contract/src/collection-name.ts diff --git a/packages/api/src/lib/__tests__/derivations-csv.test.ts b/packages/api/src/lib/__tests__/derivations-csv.test.ts index cc5330c7..97a18657 100644 --- a/packages/api/src/lib/__tests__/derivations-csv.test.ts +++ b/packages/api/src/lib/__tests__/derivations-csv.test.ts @@ -10,8 +10,11 @@ import { specimen, studySubject, micronixTube, + micronixPlate, } from '../../db/schema' -import { eq } from 'drizzle-orm' +import { eq, or } from 'drizzle-orm' +import { newCollectionName } from '@sampledb/contract' +import { runBulkCombinedImport } from '../bulk-combined-import' import { createTestStudy, createTestStudySubject, @@ -562,4 +565,116 @@ describe('derivations-csv', () => { expect(result.rows[0].error).toBe('box_name or bag_name is required for paper') }) }) + + describe('Collections named by barcode across imports', () => { + let testDb: Database + let sqlite: Awaited>['sqlite'] + + beforeEach(async () => { + const setup = await setupTestDatabase() + testDb = setup.db + sqlite = setup.sqlite + }) + + afterEach(() => { + if (sqlite) cleanupTestDatabase(sqlite) + }) + + const settings = { + derivationType: 'Extraction', + specimenTypeName: 'DNA', + containerType: 'micronix_tube' as const, + protocol: 'Standard', + derivationDate: '2025-01-15', + } + + /** A parent tube to derive from, and a study and location bulk import can register into. */ + async function seed() { + const now = utcNow() + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await setContainerDefaults(testDb, { micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' } }) + await testDb.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const study = await createTestStudy(testDb, { title: 'Deriv Study', shortCode: 'DERIV' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj1' }) + const dnaType = await createTestSpecimenType(testDb, { name: 'DNA' }) + await testDb.insert(specimenTypeContainerType).values({ specimenTypeId: dnaType.id, containerType: 'micronix_tube', created: now }) + const [parentSpecimen] = await testDb + .insert(specimen) + .values({ studySubjectId: subject.id, specimenTypeId: dnaType.id, collectionDate: '2025-01-01', created: now, lastUpdated: now }) + .returning() + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const loc = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id), canContainCollections: true }) + const parentPlate = await createTestMicronixPlate(testDb, { name: 'ParentPlate', locationId: loc.id }) + const [parent] = await testDb + .insert(storageContainer) + .values({ specimenId: parentSpecimen.id, unitId: unit.id, totalQuantity: 10, remainingQuantity: 10, created: now, lastUpdated: now }) + .returning() + await createTestMicronixTube(testDb, { containerId: parent.id, collectionId: parentPlate.id, position: 'A01', barcode: 'MT-PARENT' }) + return { parent, locationId: loc.id } + } + + /** Register one DNA tube on a plate through bulk import. */ + async function bulkImportTube(barcode: string, collection: { name?: string; barcode?: string; locationId: number; position: string }) { + return runBulkCombinedImport( + testDb, + { + studyShortCode: 'DERIV', + atomicMode: 'full_file', + subjects: [ + { + subjectName: 'Subj1', + specimens: [ + { + specimenTypeName: 'DNA', + collectionDate: '2025-02-01', + container: { containerType: 'micronix_tube', barcode, collection: { type: 'micronix_plate', ...collection } }, + }, + ], + }, + ], + }, + undefined, + ) + } + + const platesCalled = (identifier: string) => + testDb + .select({ id: micronixPlate.id, name: micronixPlate.name, barcode: micronixPlate.barcode }) + .from(micronixPlate) + .where(or(eq(micronixPlate.name, identifier), eq(micronixPlate.barcode, identifier))) + .all() + + it('finds the plate bulk import created from a barcode alone, by its name or its barcode', async () => { + const { parent, locationId } = await seed() + await bulkImportTube('MT-BULK', { barcode: 'PB-1', locationId, position: 'A01' }) + expect(platesCalled('PB-1')).toEqual([expect.objectContaining({ name: 'PB-1', barcode: 'PB-1' })]) + + const byName = await validateDerivationsCsv(testDb, ['parent_container_id,plate_name,position,container_barcode', `${parent.id},PB-1,B01,C1`].join('\n'), settings) + const byBarcode = await validateDerivationsCsv(testDb, ['parent_container_id,collection_barcode,position,container_barcode', `${parent.id},PB-1,B01,C1`].join('\n'), settings) + + expect(byName.collections).toEqual([expect.objectContaining({ name: 'PB-1', status: 'existing' })]) + expect(byBarcode.collections).toEqual([expect.objectContaining({ barcode: 'PB-1', status: 'existing' })]) + }) + + it('lets bulk import find the plate the Derivations import creates from a barcode alone', async () => { + const { parent, locationId } = await seed() + const barcodeOnlyCsv = ['parent_container_id,collection_barcode,position,container_barcode', `${parent.id},PB-2,A01,C1`].join('\n') + const validation = await validateDerivationsCsv(testDb, barcodeOnlyCsv, settings) + const [missing] = validation.collections + expect(missing).toMatchObject({ barcode: 'PB-2', status: 'will_be_created' }) + // The Derivations import wizard creates each missing Collection under this name. + await createTestMicronixPlate(testDb, { name: newCollectionName(missing)!, barcode: missing.barcode, locationId }) + const derived = await importDerivationsFromCsv(testDb, barcodeOnlyCsv, { settings }) + expect(derived.rows).toEqual([expect.objectContaining({ success: true })]) + + const byName = await bulkImportTube('MT-BULK-1', { name: 'PB-2', locationId, position: 'A02' }) + const byBarcode = await bulkImportTube('MT-BULK-2', { barcode: 'PB-2', locationId, position: 'A03' }) + + expect(byName.errors ?? []).toEqual([]) + expect(byBarcode.errors ?? []).toEqual([]) + expect(platesCalled('PB-2')).toEqual([expect.objectContaining({ name: 'PB-2', barcode: 'PB-2' })]) + const tubes = await testDb.select({ position: micronixTube.position }).from(micronixTube).where(eq(micronixTube.collectionId, platesCalled('PB-2')[0].id)).all() + expect(tubes.map((t) => t.position).sort()).toEqual(['A01', 'A02', 'A03']) + }) + }) }) diff --git a/packages/api/src/lib/specimen-registration/collections.ts b/packages/api/src/lib/specimen-registration/collections.ts index 699328b6..cdaff8f9 100644 --- a/packages/api/src/lib/specimen-registration/collections.ts +++ b/packages/api/src/lib/specimen-registration/collections.ts @@ -4,7 +4,7 @@ * same Collection; nothing is created unless nothing matches and a location is given. */ import { and, eq } from 'drizzle-orm' -import type { ContainerWriteInput } from '@sampledb/contract' +import { newCollectionName, type ContainerWriteInput } from '@sampledb/contract' import type { Database } from '../../db/client' import { sheet } from '../../db/schema' import { GRID_COLLECTION_TABLES, PARENT_COLLECTION_TABLES, type GridCollectionType, type ParentCollectionType } from './tables' @@ -173,7 +173,7 @@ export class CollectionPlanner { if (collection.locationId == null) { return { error: 'Collection not found and no location provided' } } - const newName = name ?? barcode + const newName = newCollectionName(collection) if (!newName) { return { error: `A new ${label.toLowerCase()} needs a name or a barcode.` } } diff --git a/packages/contract/src/__tests__/collection-name.test.ts b/packages/contract/src/__tests__/collection-name.test.ts new file mode 100644 index 00000000..36c44a85 --- /dev/null +++ b/packages/contract/src/__tests__/collection-name.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'bun:test' +import { newCollectionName } from '../collection-name' + +describe('newCollectionName', () => { + it('keeps a given name', () => { + expect(newCollectionName({ name: 'PLATE-001', barcode: 'BC1' })).toBe('PLATE-001') + }) + + it('names a Collection given only a barcode after the barcode', () => { + expect(newCollectionName({ barcode: 'BC1' })).toBe('BC1') + expect(newCollectionName({ name: '', barcode: 'BC1' })).toBe('BC1') + expect(newCollectionName({ name: null, barcode: 'BC1' })).toBe('BC1') + }) + + it('is undefined without a name or a barcode', () => { + expect(newCollectionName({})).toBeUndefined() + expect(newCollectionName({ name: '', barcode: '' })).toBeUndefined() + }) +}) diff --git a/packages/contract/src/collection-name.ts b/packages/contract/src/collection-name.ts new file mode 100644 index 00000000..7e994878 --- /dev/null +++ b/packages/contract/src/collection-name.ts @@ -0,0 +1,18 @@ +/** + * The name a new Collection gets, shared by every path that creates one from an import: + * Specimen registration's planner, the bulk import wizard, and the Derivations bulk import. + * + * A Collection given a name keeps it. One given only a barcode is named after the barcode, so + * the same barcode yields the same Collection name whichever import creates it, and a later + * import that names the Collection by that barcode finds it instead of creating a second one. + */ + +export type NewCollectionIdentifiers = { + name?: string | null + barcode?: string | null +} + +/** The new Collection's name, or undefined when it has neither a name nor a barcode. */ +export function newCollectionName({ name, barcode }: NewCollectionIdentifiers): string | undefined { + return name || barcode || undefined +} diff --git a/packages/contract/src/index.ts b/packages/contract/src/index.ts index 8565082c..fd1679eb 100644 --- a/packages/contract/src/index.ts +++ b/packages/contract/src/index.ts @@ -106,3 +106,4 @@ export { quantityUsedError, type ContainerQuantityBounds, } from './container-quantity' +export { newCollectionName, type NewCollectionIdentifiers } from './collection-name' diff --git a/packages/docs/src/content/docs/guides/features/derivations.md b/packages/docs/src/content/docs/guides/features/derivations.md index 2570ccd4..f7371178 100644 --- a/packages/docs/src/content/docs/guides/features/derivations.md +++ b/packages/docs/src/content/docs/guides/features/derivations.md @@ -48,6 +48,8 @@ CSV columns for the derived container: - Cryovial: `box_name` or `collection_barcode`, and `position`. `container_barcode` is optional. - Paper: `box_name` or `bag_name`, plus `sheet_name`. `sublabel` is optional. +A plate or cryovial box that doesn't exist yet is created in the **Collections** step. One given only by `collection_barcode` is named after its barcode, as in bulk import, so either import finds it later by that barcode or by that name. + For `derivation_type` in CSV, use the stored value: `aliquot` for Distribution, `dna_extraction` for DNA Extraction. The UI shows readable labels. Optional `quantity_used` and `reduce_parent_quantity` per row, or once in **Import settings**. `quantity_used` must be greater than 0 and `quantity` must be 0 or more. If you reduce quantity, the parent must have enough remaining for every row that draws from it, counted together; validation lists any row that would overdraw, and the import is rejected. diff --git a/packages/web/src/components/BulkImportFlow.tsx b/packages/web/src/components/BulkImportFlow.tsx index d171f27d..24381837 100644 --- a/packages/web/src/components/BulkImportFlow.tsx +++ b/packages/web/src/components/BulkImportFlow.tsx @@ -29,7 +29,7 @@ import { } from '../lib/bulk-import-success-message' import LocationPicker from './LocationPicker' import '../styles/storage.css' -import { parseCsv } from '@sampledb/contract' +import { newCollectionName, parseCsv } from '@sampledb/contract' export type ImportType = LibImportType type Step = 'upload' | 'collections' | 'import' @@ -327,9 +327,8 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm try { const barcode = collection.barcode || collection.collectionBarcode || undefined - // A Collection the CSV gives only by barcode is named after it, as the import itself - // does. Every missing Collection has a name or a barcode. - const name = collection.name || collection.barcode || '' + // Every missing Collection has a name or a barcode. + const name = newCollectionName(collection) ?? '' const itemCollectionType = collection.collectionType ?? defaultCollectionType if (itemCollectionType === 'micronix_plate') { diff --git a/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts b/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts index 01ac71e8..f143af32 100644 --- a/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts +++ b/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts @@ -117,6 +117,66 @@ describe('createMissingCollections', () => { ]) }) + it('names a Collection given only a barcode after the barcode, as bulk import does', async () => { + const created: Array<{ name: string; locationId: number; barcode?: string }> = [] + const events: DerivationsBulkEvent[] = [] + const gateway = stubGateway({ + createMicronixPlate: (input) => { + created.push(input) + return Promise.resolve() + }, + createCryovialBox: (input) => { + created.push(input) + return Promise.resolve() + }, + }) + + const result = await createMissingCollections( + [ + missing({ name: undefined, barcode: 'PB-1', locationId: 7 }), + missing({ name: '', barcode: 'BX-1', containerType: 'cryovial_tube', locationId: 8 }), + ], + gateway, + (e) => events.push(e), + ) + + expect(result.allSuccess).toBe(true) + expect(created).toEqual([ + { name: 'PB-1', locationId: 7, barcode: 'PB-1' }, + { name: 'BX-1', locationId: 8, barcode: 'BX-1' }, + ]) + expect(events).toContainEqual({ + type: 'COLLECTION_PATCHED', + index: 0, + patch: { status: 'success', name: 'PB-1' }, + }) + }) + + it('refuses to create a Collection with neither a name nor a barcode', async () => { + const created: string[] = [] + const events: DerivationsBulkEvent[] = [] + const gateway = stubGateway({ + createMicronixPlate: (input) => { + created.push(input.name) + return Promise.resolve() + }, + }) + + const result = await createMissingCollections( + [missing({ name: undefined, barcode: undefined, locationId: 7 })], + gateway, + (e) => events.push(e), + ) + + expect(result.allSuccess).toBe(false) + expect(created).toEqual([]) + expect(events).toContainEqual({ + type: 'COLLECTION_PATCHED', + index: 0, + patch: { status: 'error', error: 'A new collection needs a name or a barcode' }, + }) + }) + it('skips already-created entries', async () => { const created: string[] = [] const gateway = stubGateway({ diff --git a/packages/web/src/lib/derivations-bulk-import/effects.ts b/packages/web/src/lib/derivations-bulk-import/effects.ts index 13a4203f..e4dcbc43 100644 --- a/packages/web/src/lib/derivations-bulk-import/effects.ts +++ b/packages/web/src/lib/derivations-bulk-import/effects.ts @@ -1,3 +1,4 @@ +import { newCollectionName } from '@sampledb/contract' import type { BulkDerivationSettings } from '../api/derivations' import { nextStepAfterValidate } from './collections' import { parseFullCsv } from './csv' @@ -36,22 +37,18 @@ export async function validateDerivationsCsv( } } -export function collectionNameForCreate( - coll: MissingDerivationCollection, - now = Date.now(), -): string { - return coll.name || (coll.barcode ? `Collection-${coll.barcode}` : `Collection-${now}`) -} - export async function createOneMissingCollection( gateway: DerivationsBulkGateway, coll: MissingDerivationCollection, - now = Date.now(), ): Promise<{ name: string } | { error: string }> { if (!coll.locationId) { return { error: 'Location is required' } } - const name = collectionNameForCreate(coll, now) + // Named as bulk import and Specimen registration name it: after its barcode when it has no name. + const name = newCollectionName(coll) + if (!name) { + return { error: 'A new collection needs a name or a barcode' } + } try { const input = { name, locationId: coll.locationId, barcode: coll.barcode } if (coll.containerType === 'micronix_tube') { @@ -74,7 +71,6 @@ export async function createMissingCollections( missing: MissingDerivationCollection[], gateway: DerivationsBulkGateway, emit: (event: DerivationsBulkEvent) => void, - now = Date.now(), ): Promise<{ allSuccess: boolean }> { if (missing.length === 0) return { allSuccess: true } @@ -83,7 +79,7 @@ export async function createMissingCollections( const coll = missing[i] if (coll.status === 'success') continue emit({ type: 'COLLECTION_PATCHED', index: i, patch: { status: 'creating' } }) - const result = await createOneMissingCollection(gateway, coll, now) + const result = await createOneMissingCollection(gateway, coll) if ('name' in result) { emit({ type: 'COLLECTION_PATCHED', index: i, patch: { status: 'success', name: result.name } }) } else { diff --git a/packages/web/src/lib/derivations-bulk-import/index.ts b/packages/web/src/lib/derivations-bulk-import/index.ts index 39572db7..4f4b6df6 100644 --- a/packages/web/src/lib/derivations-bulk-import/index.ts +++ b/packages/web/src/lib/derivations-bulk-import/index.ts @@ -22,7 +22,6 @@ export { fileLoadedEvent, fileReadErrorEvent, validateDerivationsCsv, - collectionNameForCreate, createOneMissingCollection, createMissingCollections, importDerivationsCsv, From aa62d303575c836a81cfab23690c5fa3080eaaa0 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:44:43 -0700 Subject: [PATCH 26/54] fix(api): compare stored timestamps with bounds in the stored format (#127) Server timestamps are stored as `YYYY-MM-DD HH:MM:SS` UTC text. A few places compared them with `toISOString()` text or read them as local time. - lib/datetime: add toStoredTimestamp (utcNow uses it), daysBefore, parseStoredTimestamp (zoneless text is UTC) and utcMonthOf. - lib/sql-helpers: add timestampAtOrAfter / timestampBefore, which write the bound in the stored format. - Admin statistics "recent logins" and error-log retention use them, with an exact days x 24h cutoff and an injectable `now` for tests. - The monthly specimen timelines bucket by UTC month, so a collection date of 2024-02-01 no longer lands in January west of UTC. - timestampFilter writes an HH:MM bound as HH:MM:00, so created_to includes a row stored at exactly that minute. - The subject/control summary timeline and the activity feed sort by parsed UTC instants; reagents expiring_within_days counts from the UTC date. Tests run in America/Los_Angeles and cover rows on either side of each cutoff. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/__tests__/datetime.test.ts | 38 +++- .../lib/__tests__/timestamp-bounds.test.ts | 172 ++++++++++++++++++ packages/api/src/lib/container-enrichment.ts | 4 +- packages/api/src/lib/date-filters.ts | 8 +- packages/api/src/lib/datetime.ts | 40 +++- packages/api/src/lib/error-logger.ts | 18 +- packages/api/src/lib/sql-helpers.ts | 15 +- .../api/src/lib/statistics/admin-stats.ts | 12 +- .../src/lib/statistics/specimen-aggregates.ts | 6 +- packages/api/src/routes/activity.ts | 5 +- packages/api/src/routes/reagents.ts | 7 +- 11 files changed, 294 insertions(+), 31 deletions(-) create mode 100644 packages/api/src/lib/__tests__/timestamp-bounds.test.ts diff --git a/packages/api/src/lib/__tests__/datetime.test.ts b/packages/api/src/lib/__tests__/datetime.test.ts index 48eb498e..aae86ef7 100644 --- a/packages/api/src/lib/__tests__/datetime.test.ts +++ b/packages/api/src/lib/__tests__/datetime.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect } from 'vitest' -import { shiftCalendarDate } from '../datetime' +import { describe, it, expect, beforeAll, afterAll } from 'vitest' +import { daysBefore, parseStoredTimestamp, shiftCalendarDate, toStoredTimestamp, utcMonthOf } from '../datetime' describe('shiftCalendarDate', () => { it('rolls over months and years', () => { @@ -16,3 +16,37 @@ describe('shiftCalendarDate', () => { expect(shiftCalendarDate('1/5/2024', 1)).toBeNull() }) }) + +describe('stored timestamp helpers in a non-UTC process time zone', () => { + const originalTz = process.env.TZ + beforeAll(() => { + process.env.TZ = 'America/Los_Angeles' + }) + afterAll(() => { + if (originalTz === undefined) delete process.env.TZ + else process.env.TZ = originalTz + }) + + it('formats an instant as stored UTC text', () => { + expect(toStoredTimestamp(new Date('2024-01-15T23:59:59.999Z'))).toBe('2024-01-15 23:59:59') + }) + + it('reads zoneless stored text as UTC', () => { + expect(parseStoredTimestamp('2024-01-15 10:00:00')?.toISOString()).toBe('2024-01-15T10:00:00.000Z') + expect(parseStoredTimestamp('2024-01-15T10:00:00')?.toISOString()).toBe('2024-01-15T10:00:00.000Z') + expect(parseStoredTimestamp('2024-01-15')?.toISOString()).toBe('2024-01-15T00:00:00.000Z') + expect(parseStoredTimestamp('2024-01-15T10:00:00+02:00')?.toISOString()).toBe('2024-01-15T08:00:00.000Z') + expect(parseStoredTimestamp('not a date')).toBeNull() + }) + + it('gives the UTC month at either side of a month boundary', () => { + expect(utcMonthOf('2024-01-31 23:59:59')).toBe('2024-01') + expect(utcMonthOf('2024-02-01 00:00:00')).toBe('2024-02') + expect(utcMonthOf('2024-02-01')).toBe('2024-02') + expect(utcMonthOf('garbage')).toBeNull() + }) + + it('steps back whole 24-hour days across a DST change', () => { + expect(daysBefore(new Date('2024-03-15T12:00:00Z'), 7).toISOString()).toBe('2024-03-08T12:00:00.000Z') + }) +}) diff --git a/packages/api/src/lib/__tests__/timestamp-bounds.test.ts b/packages/api/src/lib/__tests__/timestamp-bounds.test.ts new file mode 100644 index 00000000..d960cd4e --- /dev/null +++ b/packages/api/src/lib/__tests__/timestamp-bounds.test.ts @@ -0,0 +1,172 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest' +import { and, asc, eq } from 'drizzle-orm' +import type { Database } from '../../db/client' +import { errorLogs, study, users } from '../../db/schema' +import { setupTestDatabase, cleanupTestDatabase } from '../../__tests__/helpers/db-setup' +import { createTestSpecimen, createTestSpecimenType, createTestStudy } from '../../__tests__/helpers/factories' +import { createTestUser } from '../../__tests__/helpers/auth-helpers' +import { timestampFilter } from '../date-filters' +import { buildSpecimenCollectionSummary, type EnrichedSpecimenSummary } from '../container-enrichment' +import { cleanupOldErrorLogs } from '../error-logger' +import { dateRangeConditions } from '../sql-helpers' +import { getAdminStatistics } from '../statistics/admin-stats' +import { computeSpecimenAggregates } from '../statistics/specimen-aggregates' + +// Issue #127: every bound on a stored timestamp (`YYYY-MM-DD HH:MM:SS`, UTC) is written in that +// format. Each test runs with a non-UTC process time zone so local-time mistakes show. +describe('timestamp bounds against stored timestamps', () => { + const originalTz = process.env.TZ + let db: Database + let sqlite: Awaited>['sqlite'] + + beforeAll(() => { + process.env.TZ = 'America/Los_Angeles' + }) + afterAll(() => { + if (originalTz === undefined) delete process.env.TZ + else process.env.TZ = originalTz + }) + beforeEach(async () => { + const setup = await setupTestDatabase() + db = setup.db + sqlite = setup.sqlite + }) + afterEach(() => { + cleanupTestDatabase(sqlite) + }) + + describe('admin statistics recent logins', () => { + const now = new Date('2024-03-15T12:00:00Z') // cutoff 2024-03-08 12:00:00, across the DST change + + async function userWithLastLogin(email: string, lastLogin: string | null) { + const user = await createTestUser(db, { email, name: email, role: 'member' }) + await db.update(users).set({ lastLogin }).where(eq(users.id, user.id)) + } + + it('counts logins at or after the cutoff, including later on the cutoff day', async () => { + await userWithLastLogin('before@test.com', '2024-03-08 11:59:59') + await userWithLastLogin('at@test.com', '2024-03-08 12:00:00') + await userWithLastLogin('later-same-day@test.com', '2024-03-08 18:00:00') + await userWithLastLogin('recent@test.com', '2024-03-15 11:00:00') + await userWithLastLogin('never@test.com', null) + + const data = await getAdminStatistics(db, now) + + expect(data.users.recentLogins).toBe(3) + }) + }) + + describe('error-log retention', () => { + const now = new Date('2024-03-15T12:00:00Z') // 90 days back: 2023-12-16 12:00:00 + + it('deletes exactly the rows older than the cutoff', async () => { + const stamps = [ + '2023-12-15 23:59:59', + '2023-12-16 11:59:59', + '2023-12-16 12:00:00', + '2023-12-16 18:00:00', + '2024-03-15 11:00:00', + ] + for (const timestamp of stamps) { + await db.insert(errorLogs).values({ timestamp, source: 'backend', level: 'error', message: timestamp }) + } + + const result = await cleanupOldErrorLogs(db, 90, now) + + expect(result).toEqual({ deleted: 2, retentionDays: 90 }) + const remaining = await db.select({ timestamp: errorLogs.timestamp }).from(errorLogs).orderBy(asc(errorLogs.timestamp)) + expect(remaining.map((r) => r.timestamp)).toEqual(['2023-12-16 12:00:00', '2023-12-16 18:00:00', '2024-03-15 11:00:00']) + }) + }) + + describe('monthly created timeline', () => { + it('buckets by UTC month', async () => { + const type = await createTestSpecimenType(db, { name: 'Blood' }) + // Each February row is still January 31 in Los Angeles; the UTC month is February. + const endOfJanuary = await createTestSpecimen(db, type.id, { created: '2024-01-31 23:59:59' }) + const startOfFebruary = await createTestSpecimen(db, type.id, { created: '2024-02-01 00:00:00' }) + const zonedFebruary = await createTestSpecimen(db, type.id, { created: '2024-02-01T03:00:00Z' }) + + const result = await computeSpecimenAggregates(db, [endOfJanuary, startOfFebruary, zonedFebruary], {}) + + expect(result.creationTimeline).toEqual([ + { date: '2024-01', count: 1 }, + { date: '2024-02', count: 2 }, + ]) + }) + + it('buckets a collection date by its own month', async () => { + const type = await createTestSpecimenType(db, { name: 'Blood' }) + const first = await createTestSpecimen(db, type.id, { collectionDate: '2024-02-01' }) + const last = await createTestSpecimen(db, type.id, { collectionDate: '2024-01-31' }) + + const result = await computeSpecimenAggregates(db, [first, last], {}) + + expect(result.collectionTimeline).toEqual([ + { date: '2024-01', count: 1 }, + { date: '2024-02', count: 1 }, + ]) + }) + }) + + describe('created_from / created_to filters', () => { + async function seedStudies(createdValues: string[]) { + for (const [i, created] of createdValues.entries()) { + await createTestStudy(db, { title: created, shortCode: `S${i}`, created }) + } + } + + async function titlesWithin(from: string | undefined, to: string | undefined) { + const rows = await db + .select({ title: study.title }) + .from(study) + .where(and(...dateRangeConditions(study.created, timestampFilter('created_from', from), timestampFilter('created_to', to)))) + .orderBy(asc(study.created)) + return rows.map((r) => r.title) + } + + it('includes the whole created_to day and excludes the next', async () => { + await seedStudies(['2024-01-15 23:59:59', '2024-01-16 00:00:00']) + expect(await titlesWithin(undefined, '2024-01-15')).toEqual(['2024-01-15 23:59:59']) + }) + + it('includes the start of the created_from day and excludes the day before', async () => { + await seedStudies(['2024-01-14 23:59:59', '2024-01-15 00:00:00']) + expect(await titlesWithin('2024-01-15', undefined)).toEqual(['2024-01-15 00:00:00']) + }) + + it('compares a bound with a T separator or no seconds as the stored instant', async () => { + await seedStudies(['2024-01-15 09:59:59', '2024-01-15 10:00:00', '2024-01-15 10:00:01']) + expect(await titlesWithin('2024-01-15T10:00', '2024-01-15T10:00')).toEqual(['2024-01-15 10:00:00']) + expect(await titlesWithin('2024-01-15T10:00:00', '2024-01-15 10:00:00')).toEqual(['2024-01-15 10:00:00']) + }) + }) + + describe('specimen timeline order', () => { + function summarySpecimen(id: number, collectionDate: string | null, created: string): EnrichedSpecimenSummary { + return { + id, + specimenTypeId: 1, + specimenTypeName: 'Blood', + collectionDate, + created, + lastUpdated: created, + containerCount: 0, + containerBreakdown: {}, + unitBreakdown: {}, + containers: [], + } + } + + it('orders a created timestamp and a collection date by their UTC instants', () => { + // Read as local time, 2024-01-14 20:00:00 would land after 2024-01-15 00:00 UTC. + const summary = buildSpecimenCollectionSummary( + [summarySpecimen(1, '2024-01-15', '2024-01-20 00:00:00'), summarySpecimen(2, null, '2024-01-14 20:00:00')], + [], + new Map(), + ) + + expect(summary.timeline.map((entry) => entry.id)).toEqual([2, 1]) + }) + }) +}) diff --git a/packages/api/src/lib/container-enrichment.ts b/packages/api/src/lib/container-enrichment.ts index 428ed022..f16b7839 100644 --- a/packages/api/src/lib/container-enrichment.ts +++ b/packages/api/src/lib/container-enrichment.ts @@ -1,4 +1,5 @@ import type { Database } from '../db/client' +import { parseStoredTimestamp } from './datetime' import { resolveContainerPlacementBundle, type ContainerPlacement, @@ -225,7 +226,8 @@ export function buildSpecimenCollectionSummary( specimenTypeName: spec.specimenTypeName, specimenTypeId: spec.specimenTypeId, })) - .sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()) + // A collection date (YYYY-MM-DD) and a created timestamp are both UTC; read them alike. + .sort((a, b) => (parseStoredTimestamp(a.date)?.getTime() ?? 0) - (parseStoredTimestamp(b.date)?.getTime() ?? 0)) const summary: SpecimenCollectionSummary = { totalSpecimens: enrichedSpecimens.length, diff --git a/packages/api/src/lib/date-filters.ts b/packages/api/src/lib/date-filters.ts index 244adaa6..043a640f 100644 --- a/packages/api/src/lib/date-filters.ts +++ b/packages/api/src/lib/date-filters.ts @@ -30,7 +30,8 @@ export function collectionDateFilter(label: string, value: string | null | undef /** * A bound on a created timestamp (stored as `YYYY-MM-DD HH:MM:SS`): YYYY-MM-DD, optionally - * followed by a time. A `T` separator is stored as a space so it compares with stored values. + * followed by a time. The time is written as stored (space separator, seconds) so the bound + * compares with stored values as text. * Empty or omitted means no bound; anything else throws a `ValidationError` naming `label`. */ export function timestampFilter(label: string, value: string | null | undefined): string | undefined { @@ -41,5 +42,8 @@ export function timestampFilter(label: string, value: string | null | undefined) if (!match || !isIsoCalendarDate(match[1])) { throw new ValidationError(`Invalid ${label} '${value}': use YYYY-MM-DD or YYYY-MM-DD HH:MM:SS.`) } - return match[2] ? `${match[1]} ${match[2]}` : match[1] + if (!match[2]) return match[1] + // HH:MM means HH:MM:00, so an upper bound includes a row stored at exactly that minute. + const time = match[2].length === 5 ? `${match[2]}:00` : match[2] + return `${match[1]} ${time}` } diff --git a/packages/api/src/lib/datetime.ts b/packages/api/src/lib/datetime.ts index 538aba43..31c96a8f 100644 --- a/packages/api/src/lib/datetime.ts +++ b/packages/api/src/lib/datetime.ts @@ -1,6 +1,44 @@ +const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * An instant in the format server timestamps are stored in: `YYYY-MM-DD HH:MM:SS`, UTC, the + * same text SQLite's `current_timestamp` writes. Compare a stored timestamp column only with a + * bound in this format; an ISO string (`…T…Z`) sorts after every stored value on its day. + */ +export function toStoredTimestamp(instant: Date): string { + return instant.toISOString().slice(0, 19).replace('T', ' ') +} + /** UTC timestamp in SQLite canonical format: YYYY-MM-DD HH:MM:SS */ export function utcNow(): string { - return new Date().toISOString().replace('T', ' ').replace('Z', '').split('.')[0] + return toStoredTimestamp(new Date()) +} + +/** The instant exactly `days` × 24 hours before `now`, independent of the local time zone. */ +export function daysBefore(now: Date, days: number): Date { + return new Date(now.getTime() - days * DAY_MS) +} + +/** + * The instant a stored timestamp names. A value without a zone (`YYYY-MM-DD HH:MM:SS`, + * `YYYY-MM-DDTHH:MM:SS`, or a bare `YYYY-MM-DD`) is UTC, whatever the process time zone; + * `new Date(text)` would read the first two as local time. Null when the text isn't a date. + */ +export function parseStoredTimestamp(value: string): Date | null { + const text = value.trim() + const zoneless = /^(\d{4}-\d{2}-\d{2})(?:[T ](\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?))?$/.exec(text) + let date = new Date(text) + if (zoneless) { + const [, day, time = '00:00:00'] = zoneless + date = new Date(`${day}T${time}Z`) + } + return Number.isNaN(date.getTime()) ? null : date +} + +/** The UTC month (`YYYY-MM`) of a stored timestamp or date, or null when it isn't a date. */ +export function utcMonthOf(value: string): string | null { + const date = parseStoredTimestamp(value) + return date ? date.toISOString().slice(0, 7) : null } /** diff --git a/packages/api/src/lib/error-logger.ts b/packages/api/src/lib/error-logger.ts index 5c2b129d..9bf50488 100644 --- a/packages/api/src/lib/error-logger.ts +++ b/packages/api/src/lib/error-logger.ts @@ -1,7 +1,8 @@ import type { Database } from '../db/client' import { errorLogs } from '../db/schema' -import { lt, sql } from 'drizzle-orm' -import { utcNow } from './datetime' +import { sql } from 'drizzle-orm' +import { daysBefore, utcNow } from './datetime' +import { timestampBefore } from './sql-helpers' import { logError as logObservabilityError } from './observability' function logPersistFailure( @@ -208,21 +209,18 @@ function getRetentionDays(): number { */ export async function cleanupOldErrorLogs( db: Database, - retentionDays?: number + retentionDays?: number, + now: Date = new Date(), ): Promise<{ deleted: number; retentionDays: number }> { const days = retentionDays ?? getRetentionDays() - - // Calculate cutoff date - const cutoffDate = new Date() - cutoffDate.setDate(cutoffDate.getDate() - days) - const cutoffTimestamp = cutoffDate.toISOString() + const olderThanCutoff = timestampBefore(errorLogs.timestamp, daysBefore(now, days)) try { // Count how many logs will be deleted const countResult = await db .select({ count: sql`count(*)` }) .from(errorLogs) - .where(lt(errorLogs.timestamp, cutoffTimestamp)) + .where(olderThanCutoff) const countToDelete = countResult[0]?.count || 0 @@ -230,7 +228,7 @@ export async function cleanupOldErrorLogs( if (countToDelete > 0) { await db .delete(errorLogs) - .where(lt(errorLogs.timestamp, cutoffTimestamp)) + .where(olderThanCutoff) } return { diff --git a/packages/api/src/lib/sql-helpers.ts b/packages/api/src/lib/sql-helpers.ts index 01a91da3..55bea307 100644 --- a/packages/api/src/lib/sql-helpers.ts +++ b/packages/api/src/lib/sql-helpers.ts @@ -1,6 +1,19 @@ import { gte, lt, lte, type SQL } from 'drizzle-orm' import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' -import { shiftCalendarDate } from './datetime' +import { shiftCalendarDate, toStoredTimestamp } from './datetime' + +/** + * `column >= instant` for a stored timestamp column (`YYYY-MM-DD HH:MM:SS`, UTC). The bound is + * written in the stored format, so the text comparison orders the same as the instants. + */ +export function timestampAtOrAfter(column: SQLiteColumn, instant: Date): SQL { + return gte(column, toStoredTimestamp(instant)) +} + +/** `column < instant` for a stored timestamp column; see `timestampAtOrAfter`. */ +export function timestampBefore(column: SQLiteColumn, instant: Date): SQL { + return lt(column, toStoredTimestamp(instant)) +} /** * Upper bound for a "to" date filter. A bare YYYY-MM-DD includes that whole day, so a diff --git a/packages/api/src/lib/statistics/admin-stats.ts b/packages/api/src/lib/statistics/admin-stats.ts index 2e5224c3..59b2631c 100644 --- a/packages/api/src/lib/statistics/admin-stats.ts +++ b/packages/api/src/lib/statistics/admin-stats.ts @@ -23,9 +23,11 @@ import { } from '../../db/schema' import { and, sql, isNull, gt } from 'drizzle-orm' import type { AdminStatistics } from './types' +import { daysBefore } from '../datetime' +import { timestampAtOrAfter } from '../sql-helpers' /** Admin dashboard entity and reference-data counts. */ -export async function getAdminStatistics(database: Database): Promise { +export async function getAdminStatistics(database: Database, now: Date = new Date()): Promise { const [totalUsers, activeUsers, deletedUsers, usersByRole] = await Promise.all([ database.select({ count: sql`count(*)` }).from(users), database.select({ count: sql`count(*)` }).from(users).where(isNull(users.deletedAt)), @@ -40,11 +42,11 @@ export async function getAdminStatistics(database: Database): Promise`count(*)` }) .from(sessions) - .where(gt(sessions.expiresAt, now)) + .where(gt(sessions.expiresAt, nowSeconds)) const [studiesCount, subjectsCount, specimensCount, containersCount] = await Promise.all([ database.select({ count: sql`count(*)` }).from(study), @@ -77,12 +79,10 @@ export async function getAdminStatistics(database: Database): Promise`count(*)` }).from(location) - const sevenDaysAgo = new Date() - sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7) const recentLogins = await database .select({ count: sql`count(*)` }) .from(users) - .where(and(isNull(users.deletedAt), sql`${users.lastLogin} >= ${sevenDaysAgo.toISOString()}`)) + .where(and(isNull(users.deletedAt), timestampAtOrAfter(users.lastLogin, daysBefore(now, 7)))) const usersByRoleMap: Record = {} usersByRole.forEach((row: { role: string; count: number }) => { diff --git a/packages/api/src/lib/statistics/specimen-aggregates.ts b/packages/api/src/lib/statistics/specimen-aggregates.ts index fb0e270d..7407bfba 100644 --- a/packages/api/src/lib/statistics/specimen-aggregates.ts +++ b/packages/api/src/lib/statistics/specimen-aggregates.ts @@ -3,6 +3,7 @@ import { specimen, specimenType, study, studySubject } from '../../db/schema' import { inArray } from 'drizzle-orm' import { cache, cacheKeys } from '../cache' import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' +import { utcMonthOf } from '../datetime' export type SpecimenAggregates = { total: number @@ -28,8 +29,9 @@ function buildMonthlyTimeline( .filter((s) => (dateField === 'collectionDate' ? s.collectionDate : true)) .forEach((s) => { const rawDate = dateField === 'collectionDate' ? s.collectionDate! : s.created - const date = new Date(rawDate) - const monthKey = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}` + // UTC month: `created` is a UTC timestamp and a collection date has no time of day. + const monthKey = utcMonthOf(rawDate) + if (!monthKey) return monthMap.set(monthKey, (monthMap.get(monthKey) || 0) + 1) }) diff --git a/packages/api/src/routes/activity.ts b/packages/api/src/routes/activity.ts index b12e6532..228318a0 100644 --- a/packages/api/src/routes/activity.ts +++ b/packages/api/src/routes/activity.ts @@ -13,6 +13,7 @@ import { import { sql, eq } from 'drizzle-orm' import { createAuthMiddleware } from '../middleware/auth' import { handleRouteError } from '../lib/error-handler' +import { parseStoredTimestamp } from '../lib/datetime' import { resolveSpecimenSources } from '../lib/specimens/provenance' /** Upper bound for GET /activity/recent?limit=. */ @@ -193,8 +194,8 @@ export function createActivityRoutes(database: Database): Hono { ] .filter(item => item.timestamp) // Filter out items without timestamps .sort((a, b) => { - const timeA = new Date(a.timestamp).getTime() - const timeB = new Date(b.timestamp).getTime() + const timeA = parseStoredTimestamp(a.timestamp)?.getTime() ?? 0 + const timeB = parseStoredTimestamp(b.timestamp)?.getTime() ?? 0 return timeB - timeA }) .slice(0, limit) diff --git a/packages/api/src/routes/reagents.ts b/packages/api/src/routes/reagents.ts index 6e55b57e..f3926995 100644 --- a/packages/api/src/routes/reagents.ts +++ b/packages/api/src/routes/reagents.ts @@ -5,7 +5,7 @@ import { eq, and, lte } from 'drizzle-orm' import { z } from 'zod' import { sql } from 'drizzle-orm' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' -import { utcNow } from '../lib/datetime' +import { shiftCalendarDate, utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' import { handleRouteError } from '../lib/error-handler' import { insertedRow } from '../lib/inserted-row' @@ -35,9 +35,8 @@ export function createReagentsRoutes(database: Database): Hono { if (expiringWithinDays) { const days = parseInt(expiringWithinDays) if (!isNaN(days)) { - const futureDate = new Date() - futureDate.setDate(futureDate.getDate() + days) - const dateStr = futureDate.toISOString().split('T')[0] + // expiration_date is a calendar date; count days from today's UTC date. + const dateStr = shiftCalendarDate(utcNow(), days) conditions.push( sql`${reagent.expirationDate} IS NOT NULL AND ${reagent.expirationDate} <= ${dateStr}` ) From 514c178492818d452b29157b0949021552619998 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:46:43 -0700 Subject: [PATCH 27/54] fix(specimens): page, sort, and search the Specimens list on the server (#115) GET /specimens always paginates: without page or limit it returns the first page at the configured default page size, with the total. New sort (created | collection_date) and order (asc | desc) params sort the whole filtered set before paging; undated specimens come last, and the id breaks ties so pages neither repeat nor skip rows. An unknown sort or order is a 400. The Specimens page sends page, limit, sort, and order and shows the server total. DataTable gains a server mode (pagination.totalItems, controlled sort) that neither slices nor re-sorts the rows, and memoizes its local sort. SpecimenFilter applies typed search after a 300 ms pause, so a search sends one request; Enter still applies at once. Real-data copy (118k specimens): the initial load drops from 28.2 MB in ~470 ms to 12.5 KB in ~75 ms. Co-Authored-By: Claude Opus 5.5 --- .../specimens/__tests__/specimen-read.test.ts | 97 +++++++++- .../api/src/lib/specimens/specimen-read.ts | 102 +++++++--- .../src/routes/__tests__/specimens.test.ts | 53 ++++++ packages/api/src/routes/specimens.ts | 2 + .../__tests__/helpers/mock-api-templates.ts | 4 +- packages/web/src/components/DataTable.tsx | 73 +++++--- .../web/src/components/SpecimenFilter.tsx | 24 ++- .../components/__tests__/DataTable.test.tsx | 45 +++++ .../src/hooks/__tests__/useSpecimens.test.tsx | 37 +++- packages/web/src/hooks/useSpecimens.ts | 16 +- packages/web/src/lib/api/specimens.ts | 12 +- packages/web/src/pages/Specimens.tsx | 43 ++++- .../src/pages/__tests__/Specimens.test.tsx | 174 +++++++++++++++++- 13 files changed, 591 insertions(+), 91 deletions(-) diff --git a/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts b/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts index f059576a..fc6fd66f 100644 --- a/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts +++ b/packages/api/src/lib/specimens/__tests__/specimen-read.test.ts @@ -16,6 +16,9 @@ import { createTestUnit, } from '../../../__tests__/helpers/factories' import { listSpecimens } from '../specimen-read' +import { specimen } from '../../../db/schema' +import { ValidationError } from '../../error-handler' +import { eq } from 'drizzle-orm' describe('specimen-read', () => { let testDb: Database @@ -35,15 +38,97 @@ describe('specimen-read', () => { }) describe('listSpecimens', () => { - it('returns all specimens when no pagination params provided', async () => { + it('returns the first page at the default page size, with the total, when no page or limit is given', async () => { + const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) const study = await createTestStudy(testDb, { title: 'List Study', shortCode: 'LST01' }) const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 1' }) - const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) - await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id }) + for (let day = 1; day <= 30; day++) { + await createTestSpecimen(testDb, specimenType.id, { + studySubjectId: subject.id, + collectionDate: `2024-01-${String(day).padStart(2, '0')}`, + }) + } const result = await listSpecimens(testDb, {}) - expect(result.specimens.length).toBeGreaterThanOrEqual(1) - expect(result.pagination).toBeUndefined() + + // setupPaginationSettings: defaultPageSize 25 + expect(result.specimens).toHaveLength(25) + expect(result.pagination).toEqual({ page: 1, limit: 25, total: 30, totalPages: 2 }) + }) + + it('returns the requested page, and the pages together hold every specimen once', async () => { + const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) + const study = await createTestStudy(testDb, { title: 'Page Study', shortCode: 'PG01' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 1' }) + const created = '2024-06-01 00:00:00' + const ids: number[] = [] + for (let day = 1; day <= 7; day++) { + // Equal created timestamps: the id tie-break keeps page boundaries stable. + const s = await createTestSpecimen(testDb, specimenType.id, { + studySubjectId: subject.id, + collectionDate: `2024-01-0${day}`, + created, + }) + ids.push(s.id) + } + + const pages = await Promise.all([1, 2, 3].map((page) => listSpecimens(testDb, { page: String(page), limit: '3' }))) + + expect(pages.map((p) => p.specimens.length)).toEqual([3, 3, 1]) + expect(pages[2].pagination).toEqual({ page: 3, limit: 3, total: 7, totalPages: 3 }) + expect(pages.flatMap((p) => p.specimens.map((s) => s.id))).toEqual([...ids].reverse()) + }) + + it('sorts by collection date across pages, with undated specimens last in either direction', async () => { + const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) + const study = await createTestStudy(testDb, { title: 'Sort Study', shortCode: 'SRT01' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 1' }) + const otherSubject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 2' }) + for (const date of ['2024-03-01', '2023-01-15', '2024-12-31', '2022-07-04']) { + await createTestSpecimen(testDb, specimenType.id, { studySubjectId: subject.id, collectionDate: date }) + } + const undated = await createTestSpecimen(testDb, specimenType.id, { studySubjectId: otherSubject.id }) + await testDb.update(specimen).set({ collectionDate: null }).where(eq(specimen.id, undated.id)) + + const collect = async (order: string) => { + const dates: Array = [] + for (const page of ['1', '2', '3']) { + const result = await listSpecimens(testDb, { sort: 'collection_date', order, page, limit: '2' }) + dates.push(...result.specimens.map((s) => s.collectionDate)) + } + return dates + } + + expect(await collect('asc')).toEqual(['2022-07-04', '2023-01-15', '2024-03-01', '2024-12-31', null]) + expect(await collect('desc')).toEqual(['2024-12-31', '2024-03-01', '2023-01-15', '2022-07-04', null]) + }) + + it('sorts by created, oldest first with order=asc', async () => { + const specimenType = await createTestSpecimenType(testDb, { name: 'Plasma' }) + const study = await createTestStudy(testDb, { title: 'Created Study', shortCode: 'CRT01' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 1' }) + for (const [day, created] of ['2024-05-02 08:00:00', '2024-05-01 08:00:00', '2024-05-03 08:00:00'].entries()) { + await createTestSpecimen(testDb, specimenType.id, { + studySubjectId: subject.id, + collectionDate: `2024-01-0${day + 1}`, + created, + }) + } + + const asc = await listSpecimens(testDb, { sort: 'created', order: 'asc' }) + const byDefault = await listSpecimens(testDb, {}) + + expect(asc.specimens.map((s) => s.created)).toEqual(['2024-05-01 08:00:00', '2024-05-02 08:00:00', '2024-05-03 08:00:00']) + expect(byDefault.specimens.map((s) => s.created)).toEqual(['2024-05-03 08:00:00', '2024-05-02 08:00:00', '2024-05-01 08:00:00']) + }) + + it.each([ + [{ sort: 'study' }, "Invalid sort 'study'"], + [{ sort: 'created; DROP TABLE specimen' }, 'Invalid sort'], + [{ sort: 'created', order: 'sideways' }, "Invalid order 'sideways'"], + ])('rejects %j with a ValidationError', async (params, message) => { + await expect(listSpecimens(testDb, params)).rejects.toThrow(ValidationError) + await expect(listSpecimens(testDb, params)).rejects.toThrow(message) }) it('filters specimens by study short code', async () => { @@ -88,7 +173,7 @@ describe('specimen-read', () => { it('returns empty result with default pagination for unknown barcode', async () => { const result = await listSpecimens(testDb, { barcode: 'NONEXISTENT-BARCODE' }) expect(result.specimens).toEqual([]) - expect(result.pagination).toEqual({ page: 1, limit: 50, total: 0, totalPages: 0 }) + expect(result.pagination).toEqual({ page: 1, limit: 25, total: 0, totalPages: 0 }) }) }) }) diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 01c7d52e..c6243597 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -1,7 +1,8 @@ import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject, study, specimenType, controlBatch } from '../../db/schema' -import { eq, and, or, sql, inArray } from 'drizzle-orm' +import { eq, and, or, sql, inArray, asc, desc } from 'drizzle-orm' import { validatePage, validateLimit } from '../constants' +import { ValidationError } from '../error-handler' import { findContainersByBarcode } from '../identifier-resolution' import { collectionDateFilter, timestampFilter } from '../date-filters' import { dateRangeConditions } from '../sql-helpers' @@ -21,6 +22,53 @@ export type ListSpecimensQuery = { search?: string page?: string limit?: string + sort?: string + order?: string +} + +/** Columns the Specimens list can sort by, as the `sort` query parameter names them. */ +export const SPECIMEN_SORT_COLUMNS = ['created', 'collection_date'] as const +export type SpecimenSortColumn = (typeof SPECIMEN_SORT_COLUMNS)[number] +export type SortOrder = 'asc' | 'desc' + +/** + * The list order from `sort` and `order`. With neither, newest created first; otherwise `sort` + * defaults to `created` and `order` to `asc`. An unknown column or direction throws a + * `ValidationError` (400) instead of being ignored. + */ +export function parseSpecimenSort( + sortParam: string | undefined, + orderParam: string | undefined, +): { sort: SpecimenSortColumn; order: SortOrder } { + const sortText = sortParam?.trim() ?? '' + const orderText = orderParam?.trim().toLowerCase() ?? '' + if (sortText === '' && orderText === '') return { sort: 'created', order: 'desc' } + + const sort = sortText === '' ? 'created' : sortText + if (!isSpecimenSortColumn(sort)) { + throw new ValidationError(`Invalid sort '${sortParam}': use one of ${SPECIMEN_SORT_COLUMNS.join(', ')}.`) + } + const order = orderText === '' ? 'asc' : orderText + if (order !== 'asc' && order !== 'desc') { + throw new ValidationError(`Invalid order '${orderParam}': use asc or desc.`) + } + return { sort, order } +} + +function isSpecimenSortColumn(value: string): value is SpecimenSortColumn { + return (SPECIMEN_SORT_COLUMNS as readonly string[]).includes(value) +} + +/** + * ORDER BY for a sort. Specimens without a collection date come last in either direction, + * and the id breaks ties so that consecutive pages neither repeat nor skip a row. + */ +function specimenOrderBy(sort: SpecimenSortColumn, order: SortOrder) { + const direction = order === 'asc' ? asc : desc + if (sort === 'collection_date') { + return [sql`${specimen.collectionDate} IS NULL`, direction(specimen.collectionDate), direction(specimen.id)] + } + return [direction(specimen.created), direction(specimen.id)] } export type ListSpecimensResult = { @@ -48,7 +96,7 @@ export type ListSpecimensResult = { name: string | null } | null }> - pagination?: { + pagination: { page: number limit: number total: number @@ -56,12 +104,10 @@ export type ListSpecimensResult = { } } -const emptyBarcodeResult = (): ListSpecimensResult => ({ - specimens: [], - pagination: { page: 1, limit: 50, total: 0, totalPages: 0 }, -}) - -/** List and filter specimens with optional pagination. */ +/** + * List and filter specimens one page at a time. Without `page` or `limit` it returns the + * first page at the configured default page size, with the total across all pages. + */ export async function listSpecimens(database: Database, query: ListSpecimensQuery): Promise { const { sourceType, @@ -77,8 +123,15 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer search, page: pageParam, limit: limitParam, + sort: sortParam, + order: orderParam, } = query + const page = validatePage(pageParam) + const limit = await validateLimit(database, limitParam) + const offset = (page - 1) * limit + const { sort, order } = parseSpecimenSort(sortParam, orderParam) + let selectQuery = database .select({ id: specimen.id, @@ -158,7 +211,9 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer if (barcode) { // Every Specimen with a Container carrying the barcode; cryovial barcodes repeat. const containerIds = (await findContainersByBarcode(database, barcode)).map((match) => match.containerId) - if (containerIds.length === 0) return emptyBarcodeResult() + if (containerIds.length === 0) { + return { specimens: [], pagination: { page, limit, total: 0, totalPages: 0 } } + } conditions.push( inArray( specimen.id, @@ -182,19 +237,6 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer selectQuery = selectQuery.where(and(...conditions) as any) as any } - const returnAll = !pageParam && !limitParam - - const page = pageParam ? validatePage(pageParam) : 1 - let limit: number | undefined - if (returnAll) { - limit = undefined - } else if (limitParam) { - limit = await validateLimit(database, limitParam) - } else { - limit = 50 - } - const offset = returnAll ? undefined : (page - 1) * limit! - const countQuery = database .select({ count: sql`COUNT(*)` }) .from(specimen) @@ -207,13 +249,13 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer countQuery.where(and(...conditions) as any) as any } - let queryWithOrder = selectQuery.orderBy(sql`${specimen.created} DESC`) - if (!returnAll) { - queryWithOrder = queryWithOrder.limit(limit!).offset(offset!) as any - } + const pageQuery = selectQuery + .orderBy(...specimenOrderBy(sort, order)) + .limit(limit) + .offset(offset) const [specimensList, countResult] = await Promise.all([ - queryWithOrder, + pageQuery, countQuery, ]) @@ -221,11 +263,11 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer return { specimens: specimensList, - pagination: returnAll ? undefined : { + pagination: { page, - limit: limit!, + limit, total, - totalPages: Math.ceil(total / limit!), + totalPages: Math.ceil(total / limit), }, } } diff --git a/packages/api/src/routes/__tests__/specimens.test.ts b/packages/api/src/routes/__tests__/specimens.test.ts index 1eb864d1..4564c24e 100644 --- a/packages/api/src/routes/__tests__/specimens.test.ts +++ b/packages/api/src/routes/__tests__/specimens.test.ts @@ -95,6 +95,59 @@ describe('Specimens API', () => { }) }) + describe('GET /api/specimens pagination and sort', () => { + it('returns the first page and a total when no page or limit is given', async () => { + const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) + for (let day = 1; day <= 60; day++) { + const date = new Date(Date.UTC(2024, 0, day)).toISOString().slice(0, 10) + await createTestSpecimen(ctx.db, type.id, { collectionDate: date }) + } + + const res = await ctx.request('/api/specimens') + + expect(res.status).toBe(200) + const data = (await res.json()) as { + specimens: unknown[] + pagination: { page: number; limit: number; total: number; totalPages: number } + } + expect(data.pagination.page).toBe(1) + expect(data.pagination.total).toBe(60) + expect(data.pagination.limit).toBeLessThan(60) + expect(data.specimens).toHaveLength(data.pagination.limit) + expect(data.pagination.totalPages).toBe(Math.ceil(60 / data.pagination.limit)) + }) + + it('sorts the whole filtered set before paging', async () => { + const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) + for (const date of ['2024-03-01', '2021-01-01', '2024-12-31', '2023-06-15']) { + await createTestSpecimen(ctx.db, type.id, { collectionDate: date }) + } + + const res = await ctx.request( + `/api/specimens?specimen_type_id=${type.id}&sort=collection_date&order=asc&page=2&limit=2`, + ) + + expect(res.status).toBe(200) + const data = (await res.json()) as { + specimens: Array<{ collectionDate: string }> + pagination: { total: number } + } + expect(data.specimens.map((s) => s.collectionDate)).toEqual(['2024-03-01', '2024-12-31']) + expect(data.pagination.total).toBe(4) + }) + + it.each([ + ['sort=study', 'sort'], + ['sort=created&order=up', 'order'], + ])('returns 400 for %s', async (query, param) => { + const res = await ctx.request(`/api/specimens?${query}`) + + expect(res.status).toBe(400) + const data = (await res.json()) as { error: string } + expect(data.error).toContain(`Invalid ${param}`) + }) + }) + describe('POST /api/specimens/bulk', () => { let testStudy: Awaited> let testSubject: Awaited> diff --git a/packages/api/src/routes/specimens.ts b/packages/api/src/routes/specimens.ts index 67715734..8d0af94f 100644 --- a/packages/api/src/routes/specimens.ts +++ b/packages/api/src/routes/specimens.ts @@ -42,6 +42,8 @@ specimens.get('/', authMiddleware, async (c) => { search: c.req.query('search'), page: c.req.query('page'), limit: c.req.query('limit'), + sort: c.req.query('sort'), + order: c.req.query('order'), })) } catch (error) { return handleRouteError(error, c) diff --git a/packages/web/src/__tests__/helpers/mock-api-templates.ts b/packages/web/src/__tests__/helpers/mock-api-templates.ts index 4fd237dc..06582817 100644 --- a/packages/web/src/__tests__/helpers/mock-api-templates.ts +++ b/packages/web/src/__tests__/helpers/mock-api-templates.ts @@ -126,7 +126,7 @@ export function dashboardPageMock(): MockApiOverrides { get: vi.fn().mockImplementation((url: string) => { const emptyList = { pagination: { total: 0 }, studies: [], specimens: [], subjects: [], containers: [], locations: [] } if (url === '/studies') return Promise.resolve({ studies: [], pagination: { total: 0 } }) - if (url === '/specimens') return Promise.resolve({ specimens: [], pagination: { total: 0 } }) + if (url === '/specimens') return Promise.resolve({ specimens: [], pagination: { page: 1, limit: 1, total: 0, totalPages: 0 } }) if (url === '/subjects') return Promise.resolve({ subjects: [], pagination: { total: 0 } }) if (url === '/containers') return Promise.resolve({ containers: [], pagination: { total: 0 } }) if (url === '/locations') return Promise.resolve({ locations: [], pagination: { total: 0 } }) @@ -148,7 +148,7 @@ export function specimensPageMock(): MockApiOverrides { return { default: { get: vi.fn() }, specimensApi: { - search: vi.fn().mockResolvedValue({ specimens: [] }), + search: vi.fn().mockResolvedValue({ specimens: [], pagination: { page: 1, limit: 50, total: 0, totalPages: 0 } }), }, specimenTypesApi: { list: vi.fn().mockResolvedValue({ data: [] }), diff --git a/packages/web/src/components/DataTable.tsx b/packages/web/src/components/DataTable.tsx index 99496e25..010f16ab 100644 --- a/packages/web/src/components/DataTable.tsx +++ b/packages/web/src/components/DataTable.tsx @@ -1,7 +1,7 @@ -import { useState, useEffect, useRef } from 'react' +import { useState, useEffect, useMemo, useRef } from 'react' import { SkeletonTable } from '../ui' import Pagination from './Pagination' -import { compareBySortType, type SortType } from '../lib/sort-compare' +import { compareBySortType, type SortDirection, type SortType } from '../lib/sort-compare' export interface Column { key: keyof T | string @@ -25,6 +25,20 @@ interface DataTableProps { pageSize: number onPageChange: (page: number) => void showPagination?: boolean + /** + * Rows across all pages when the server pages the list. With it, `data` is the current + * page as the server returned it, and the table does not slice it. + */ + totalItems?: number + } + /** + * Controlled sort for a list the server sorts. With it, the table shows `sort` and reports + * header clicks to `onChange`, and leaves `data` in the order it was given. + */ + sort?: { + column: string | null + direction: SortDirection + onChange: (column: string, direction: SortDirection) => void } /** Optional class for the root wrapper (e.g. dashboard-card for themed pages). */ className?: string @@ -40,10 +54,14 @@ export default function DataTable({ initialSortDirection = 'asc', density = 'normal', pagination, + sort, className, }: DataTableProps) { - const [sortColumn, setSortColumn] = useState(initialSortColumn) - const [sortDirection, setSortDirection] = useState<'asc' | 'desc'>(initialSortDirection) + const [localSortColumn, setSortColumn] = useState(initialSortColumn) + const [localSortDirection, setSortDirection] = useState(initialSortDirection) + const sortColumn = sort ? sort.column : localSortColumn + const sortDirection = sort ? sort.direction : localSortDirection + const serverPaged = pagination?.totalItems !== undefined const [selectedRowIndex, setSelectedRowIndex] = useState(null) const tableRef = useRef(null) const rowRefs = useRef>(new Map()) @@ -51,29 +69,36 @@ export default function DataTable({ const handleSort = (column: Column) => { if (!column.sortType) return - - if (sortColumn === column.key) { - setSortDirection(sortDirection === 'asc' ? 'desc' : 'asc') - } else { - setSortColumn(column.key) - setSortDirection('asc') + + const direction: SortDirection = sortColumn === column.key && sortDirection === 'asc' ? 'desc' : 'asc' + if (sort) { + sort.onChange(String(column.key), direction) + return } + setSortColumn(column.key) + setSortDirection(direction) } const sortType = columns.find((column) => column.key === sortColumn)?.sortType - const sortedData = - sortColumn && sortType - ? [...data].sort((a, b) => - compareBySortType(a[sortColumn as keyof T], b[sortColumn as keyof T], sortType, sortDirection) - ) - : data - - // Apply pagination if enabled - const paginatedData = pagination - ? sortedData.slice((pagination.page - 1) * pagination.pageSize, pagination.page * pagination.pageSize) - : sortedData - - const totalPages = pagination ? Math.ceil(sortedData.length / pagination.pageSize) : 1 + // A server-sorted list arrives in order; sort locally only when the table owns the sort. + const sortedData = useMemo( + () => + !sort && sortColumn && sortType + ? [...data].sort((a, b) => + compareBySortType(a[sortColumn as keyof T], b[sortColumn as keyof T], sortType, sortDirection) + ) + : data, + [data, sort, sortColumn, sortType, sortDirection] + ) + + // Apply pagination if enabled; a server-paged list is already one page. + const paginatedData = + pagination && !serverPaged + ? sortedData.slice((pagination.page - 1) * pagination.pageSize, pagination.page * pagination.pageSize) + : sortedData + + const totalItems = pagination?.totalItems ?? sortedData.length + const totalPages = pagination ? Math.ceil(totalItems / pagination.pageSize) : 1 // Reset selected row when data changes, sort changes, or page changes (during render to avoid extra pass) const resetDeps = [data.length, sortColumn, sortDirection, pagination?.page ?? 0] @@ -240,7 +265,7 @@ export default function DataTable({ currentPage={pagination.page} totalPages={totalPages} onPageChange={pagination.onPageChange} - totalItems={sortedData.length} + totalItems={totalItems} itemsPerPage={pagination.pageSize} />
diff --git a/packages/web/src/components/SpecimenFilter.tsx b/packages/web/src/components/SpecimenFilter.tsx index 3fa0b6a8..9edb63f9 100644 --- a/packages/web/src/components/SpecimenFilter.tsx +++ b/packages/web/src/components/SpecimenFilter.tsx @@ -19,7 +19,10 @@ export interface SpecimenFilters { interface SpecimenFilterProps { /** The applied filters. */ filters: SpecimenFilters - /** Search text applies as it is typed; other filter edits wait for Apply. */ + /** + * Search text applies once typing pauses for `SPECIMEN_SEARCH_DEBOUNCE_MS`, so a search sends + * one request instead of one per keystroke; other filter edits wait for Apply. + */ onSearchChange: (search: string | undefined) => void /** Apply the edited filters (Apply, Enter in search, or Clear). */ onSubmit: (filters: SpecimenFilters) => void @@ -41,6 +44,9 @@ function sameFilters(a: SpecimenFilters, b: SpecimenFilters, ignore?: keyof Spec return FILTER_KEYS.every((key) => key === ignore || (a[key] || undefined) === (b[key] || undefined)) } +/** How long search typing must pause before the search applies. */ +export const SPECIMEN_SEARCH_DEBOUNCE_MS = 300 + const SOURCE_TYPES = [ { value: '', label: 'All Sources' }, { value: 'subject', label: 'Subject' }, @@ -65,6 +71,20 @@ export default function SpecimenFilter({ filters, onSearchChange, onSubmit, isLo setLocalFilters(searchOnly ? { ...localFilters, search: filters.search } : filters) } + // Apply typed search text once typing pauses. Apply, Enter, Clear, or an outside change to the + // applied search makes the two equal, which cancels a pending apply. + const onSearchChangeRef = useRef(onSearchChange) + useEffect(() => { + onSearchChangeRef.current = onSearchChange + }) + const typedSearch = localFilters.search || undefined + const appliedSearch = filters.search || undefined + useEffect(() => { + if (typedSearch === appliedSearch) return + const timer = setTimeout(() => onSearchChangeRef.current(typedSearch), SPECIMEN_SEARCH_DEBOUNCE_MS) + return () => clearTimeout(timer) + }, [typedSearch, appliedSearch]) + useEffect(() => { loadReferenceData() }, []) @@ -131,7 +151,6 @@ export default function SpecimenFilter({ filters, onSearchChange, onSubmit, isLo newFilters[key] = value } setLocalFilters(newFilters) - if (key === 'search') onSearchChange(newFilters.search) } const handleSubmit = () => { @@ -165,6 +184,7 @@ export default function SpecimenFilter({ filters, onSearchChange, onSubmit, isLo updateFilter('search', e.target.value)} onKeyDown={(e) => e.key === 'Enter' && handleSubmit()} diff --git a/packages/web/src/components/__tests__/DataTable.test.tsx b/packages/web/src/components/__tests__/DataTable.test.tsx index da0761b2..5872f781 100644 --- a/packages/web/src/components/__tests__/DataTable.test.tsx +++ b/packages/web/src/components/__tests__/DataTable.test.tsx @@ -199,4 +199,49 @@ describe('DataTable', () => { expect(columnValues(0)).toEqual(subjects.map((s) => s.name)) }) }) + + describe('server-paged and server-sorted lists', () => { + const page2: TestItem[] = [ + { id: 53, name: 'Item 53', value: 3 }, + { id: 51, name: 'Item 51', value: 1 }, + { id: 52, name: 'Item 52', value: 2 }, + ] + + function names(): string[] { + return screen + .getAllByRole('row') + .slice(1) + .map((row) => within(row).getAllByRole('cell')[1].textContent) + } + + it('shows the given page as is and counts pages from totalItems', async () => { + await render( + + ) + + expect(names()).toEqual(['Item 53', 'Item 51', 'Item 52']) + expect(screen.getByText(/Showing/)).toHaveTextContent('Showing 4 to 6 of 8 results') + }) + + it('reports a header click to sort.onChange and leaves the rows in server order', async () => { + const user = userEvent.setup() + const onChange = vi.fn() + const { rerender } = await render( + + ) + + await user.click(screen.getByRole('columnheader', { name: 'Name' })) + expect(onChange).toHaveBeenLastCalledWith('name', 'asc') + expect(names()).toEqual(['Item 53', 'Item 51', 'Item 52']) + + rerender() + await user.click(screen.getByRole('columnheader', { name: 'Name' })) + expect(onChange).toHaveBeenLastCalledWith('name', 'desc') + expect(names()).toEqual(['Item 53', 'Item 51', 'Item 52']) + }) + }) }) diff --git a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx index a42373c8..f627ba43 100644 --- a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx +++ b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx @@ -48,17 +48,18 @@ describe('useSpecimens Hooks', () => { }) describe('useSpecimens', () => { - it('should fetch and return specimens', async () => { + it('returns one page of specimens with the pagination totals', async () => { const mockData = { specimens: [ { id: 1, specimenTypeId: 1, created: '2024-01-01', lastUpdated: '2024-01-01' }, { id: 2, specimenTypeId: 1, created: '2024-01-01', lastUpdated: '2024-01-01' }, ], + pagination: { page: 1, limit: 2, total: 9, totalPages: 5 }, } vi.mocked(specimensApi.search).mockResolvedValue(mockData) - const { result } = renderHook(() => useSpecimens(), { + const { result } = renderHook(() => useSpecimens({ page: 1, limit: 2 }), { wrapper: createWrapper(), }) @@ -66,16 +67,40 @@ describe('useSpecimens Hooks', () => { expect(result.current.isSuccess).toBe(true) }) - expect(result.current.data).toEqual(mockData.specimens) - expect(specimensApi.search).toHaveBeenCalledWith(undefined) + expect(result.current.data).toEqual(mockData) + expect(specimensApi.search).toHaveBeenCalledWith({ page: 1, limit: 2 }) + }) + + it('keeps the previous page on screen while the next page loads', async () => { + const page1 = { + specimens: [{ id: 1, specimenTypeId: 1, created: '2024-01-01', lastUpdated: '2024-01-01' }], + pagination: { page: 1, limit: 1, total: 2, totalPages: 2 }, + } + let resolvePage2: (value: typeof page1) => void = () => {} + vi.mocked(specimensApi.search) + .mockResolvedValueOnce(page1) + .mockReturnValueOnce(new Promise((resolve) => (resolvePage2 = resolve))) + + const { result, rerender } = renderHook(({ page }) => useSpecimens({ page, limit: 1 }), { + wrapper: createWrapper(), + initialProps: { page: 1 }, + }) + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + + rerender({ page: 2 }) + + await waitFor(() => expect(result.current.isPlaceholderData).toBe(true)) + expect(result.current.data).toEqual(page1) + resolvePage2({ ...page1, pagination: { ...page1.pagination, page: 2 } }) + await waitFor(() => expect(result.current.data?.pagination.page).toBe(2)) }) it('should pass filters to search', async () => { - const mockData = { specimens: [] } + const mockData = { specimens: [], pagination: { page: 1, limit: 50, total: 0, totalPages: 0 } } vi.mocked(specimensApi.search).mockResolvedValue(mockData) - const filters = { source_type: 'subject', study: 'ST1' } + const filters = { source_type: 'subject', study: 'ST1', page: 1, limit: 50 } const { result } = renderHook(() => useSpecimens(filters), { wrapper: createWrapper(), }) diff --git a/packages/web/src/hooks/useSpecimens.ts b/packages/web/src/hooks/useSpecimens.ts index 62f39796..4d542b7f 100644 --- a/packages/web/src/hooks/useSpecimens.ts +++ b/packages/web/src/hooks/useSpecimens.ts @@ -1,4 +1,4 @@ -import { useQuery, useMutation, useQueryClient, type QueryClient } from '@tanstack/react-query' +import { useQuery, useMutation, useQueryClient, keepPreviousData, type QueryClient } from '@tanstack/react-query' import { specimensApi, type SpecimenListParams } from '../lib/api/specimens' import { containersApi } from '../lib/api/containers' import { subjectsApi } from '../lib/api/subjects' @@ -97,13 +97,15 @@ export function invalidateSpecimenQueries( } } -export function useSpecimens(filters?: SpecimenListParams) { +/** + * One page of specimens and the pagination totals. The previous page stays on screen while + * the next page, sort, or filter loads. + */ +export function useSpecimens(params: SpecimenListParams) { return useQuery({ - queryKey: specimenKeys.list(filters), - queryFn: async () => { - const res = await specimensApi.search(filters) - return res.specimens - }, + queryKey: specimenKeys.list(params), + queryFn: () => specimensApi.search(params), + placeholderData: keepPreviousData, }) } diff --git a/packages/web/src/lib/api/specimens.ts b/packages/web/src/lib/api/specimens.ts index 52a6cc05..4e67675c 100644 --- a/packages/web/src/lib/api/specimens.ts +++ b/packages/web/src/lib/api/specimens.ts @@ -9,7 +9,11 @@ type BulkSpecimenContainerWriteInput = ContainerWriteInput & { comment?: string } -type SpecimensListResponse = { specimens: Specimen[]; pagination?: { total: number } } +/** One page of `GET /specimens`; the API always paginates. */ +export type SpecimensListResponse = { + specimens: Specimen[] + pagination: { page: number; limit: number; total: number; totalPages: number } +} type SpecimenResponse = { specimen: Specimen } type SpecimensBulkResponse = { specimens: Specimen[] @@ -57,8 +61,14 @@ export type SpecimenListParams = { subject_id?: string page?: number limit?: number + /** Server-side order across all pages; omitted means newest created first. */ + sort?: SpecimenSortColumn + order?: 'asc' | 'desc' } +/** Columns `GET /specimens` sorts by. */ +export type SpecimenSortColumn = 'created' | 'collection_date' + export const specimensApi = { search: (params?: SpecimenListParams) => api.get('/specimens', { params }), get: (id: number) => api.get(`/specimens/${id}`), diff --git a/packages/web/src/pages/Specimens.tsx b/packages/web/src/pages/Specimens.tsx index 03b4ba08..cc74e737 100644 --- a/packages/web/src/pages/Specimens.tsx +++ b/packages/web/src/pages/Specimens.tsx @@ -4,7 +4,8 @@ import { Link, useSearchParams, useNavigate } from 'react-router-dom' import SpecimenFilter, { type SpecimenFilters } from '../components/SpecimenFilter' import { useUser } from '../contexts/UserContext' import { useSpecimens } from '../hooks/useSpecimens' -import type { SpecimenListParams } from '../lib/api/specimens' +import type { SpecimenListParams, SpecimenSortColumn } from '../lib/api/specimens' +import type { SortDirection } from '../lib/sort-compare' import type { Specimen } from '../lib/api/types' import { AsyncPresentation, @@ -23,6 +24,16 @@ type SpecimenListRow = Specimen & { controlBatch?: { name: string } } +const PAGE_SIZE = 50 + +/** The table columns the API can sort the whole list by, and the `sort` value for each. */ +const SERVER_SORT_COLUMNS: Record = { + collectionDate: 'collection_date', + created: 'created', +} + +type TableSort = { column: string; direction: SortDirection } + function filtersToParams(filters: SpecimenFilters): SpecimenListParams { return { study: filters.study, @@ -54,7 +65,8 @@ export default function Specimens() { const { canWrite } = useUser() const [searchParams, setSearchParams] = useSearchParams() const [page, setPage] = useState(1) - const pageSize = 50 + // Null: the API's default order, newest created first. + const [sort, setSort] = useState(null) // The URL is the source of truth, so "Clear Filters" and back/forward update the table. const filters = useMemo(() => filtersFromSearchParams(searchParams), [searchParams]) @@ -63,10 +75,25 @@ export default function Specimens() { setPage(1) }, [searchParams]) - const listParams = useMemo(() => filtersToParams(filters), [filters]) + // The server filters, sorts, and pages, so sorting and filters cover every page. + const listParams = useMemo( + () => ({ + ...filtersToParams(filters), + page, + limit: PAGE_SIZE, + ...(sort ? { sort: SERVER_SORT_COLUMNS[sort.column], order: sort.direction } : {}), + }), + [filters, page, sort] + ) const specimensQuery = useSpecimens(listParams) - const specimens = (specimensQuery.data ?? []) as SpecimenListRow[] + const specimens = (specimensQuery.data?.specimens ?? []) as SpecimenListRow[] + const total = specimensQuery.data?.pagination.total ?? 0 + + const handleSortChange = (column: string, direction: SortDirection) => { + setSort({ column, direction }) + setPage(1) + } const listStatus = fromQuery(specimensQuery, { isEmpty: specimensQuery.isSuccess && specimens.length === 0, }) @@ -215,9 +242,15 @@ export default function Specimens() { emptyMessage="No specimens found" pagination={{ page, - pageSize, + pageSize: PAGE_SIZE, onPageChange: setPage, showPagination: true, + totalItems: total, + }} + sort={{ + column: sort?.column ?? null, + direction: sort?.direction ?? 'asc', + onChange: handleSortChange, }} className="dashboard-card overflow-hidden" /> diff --git a/packages/web/src/pages/__tests__/Specimens.test.tsx b/packages/web/src/pages/__tests__/Specimens.test.tsx index cbcd6cf9..8d2b6697 100644 --- a/packages/web/src/pages/__tests__/Specimens.test.tsx +++ b/packages/web/src/pages/__tests__/Specimens.test.tsx @@ -1,7 +1,10 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { useLocation, useNavigate } from 'react-router-dom' -import { render, screen, fireEvent, waitFor } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' +import { render, screen, fireEvent, waitFor, within } from '../../__tests__/helpers/render' import Specimens from '../Specimens' +import { SPECIMEN_SEARCH_DEBOUNCE_MS } from '../../components/SpecimenFilter' +import type { SpecimenListParams } from '../../lib/api/specimens' function BackButton() { const navigate = useNavigate() @@ -68,9 +71,41 @@ vi.mock('../../lib/api/studies', async () => { import { api } from '../../lib/api/client' +const emptyPage = { specimens: [], pagination: { page: 1, limit: 50, total: 0, totalPages: 0 } } + +/** One server page of `count` rows numbered from `firstId`, out of `total`. */ +function specimenPage(page: number, total: number, count = 50, firstId = (page - 1) * 50 + 1) { + return { + specimens: Array.from({ length: count }, (_, i) => ({ + id: firstId + i, + specimenTypeId: 1, + created: '2024-01-01', + collectionDate: '2024-01-01', + specimenType: { id: 1, name: 'Whole Blood' }, + studySubject: { id: firstId + i, name: `SUBJ-${firstId + i}` }, + study: { id: 1, shortCode: 'ST1' }, + })), + pagination: { page, limit: 50, total, totalPages: Math.ceil(total / 50) }, + } +} + +/** The params of every `GET /specimens` request so far. */ +function specimenRequests(): SpecimenListParams[] { + return vi + .mocked(api.get) + .mock.calls.filter(([url]) => url === '/specimens') + .map(([, config]) => (config as { params: SpecimenListParams }).params) +} + +function lastSpecimenRequest(): SpecimenListParams | undefined { + const requests = specimenRequests() + return requests[requests.length - 1] +} + describe('Specimens page', () => { beforeEach(() => { - vi.mocked(api.get).mockResolvedValue({ specimens: [] }) + vi.mocked(api.get).mockReset() + vi.mocked(api.get).mockResolvedValue(emptyPage) }) it('renders filters and table area', async () => { @@ -91,6 +126,7 @@ describe('Specimens page', () => { study: { id: 1, shortCode: 'ST1' }, }, ], + pagination: { page: 1, limit: 50, total: 1, totalPages: 1 }, }) await render() await screen.findByText(/Whole Blood/i) @@ -98,9 +134,114 @@ describe('Specimens page', () => { }) }) +describe('Specimens page pagination and sort', () => { + beforeEach(() => { + vi.mocked(api.get).mockReset() + }) + + it('requests one page on load and shows the server total', async () => { + vi.mocked(api.get).mockResolvedValue(specimenPage(1, 120)) + + await render() + + await screen.findByText('SUBJ-1') + expect(specimenRequests()).toEqual([expect.objectContaining({ page: 1, limit: 50 })]) + expect(specimenRequests()[0]).not.toHaveProperty('sort') + expect(screen.getByText(/Showing/)).toHaveTextContent('Showing 1 to 50 of 120 results') + }) + + it('requests the next page from the server instead of slicing the loaded rows', async () => { + const user = userEvent.setup() + vi.mocked(api.get).mockImplementation((_url, config) => { + const { page } = (config as { params: SpecimenListParams }).params + return Promise.resolve(page === 2 ? specimenPage(2, 120) : specimenPage(1, 120)) + }) + await render() + await screen.findByText('SUBJ-1') + + await user.click(screen.getByRole('button', { name: 'Next' })) + + await screen.findByText('SUBJ-51') + expect(lastSpecimenRequest()).toEqual(expect.objectContaining({ page: 2, limit: 50 })) + expect(screen.getByText(/Showing/)).toHaveTextContent('Showing 51 to 100 of 120 results') + }) + + it('sorts on the server across all pages and goes back to page 1', async () => { + const user = userEvent.setup() + vi.mocked(api.get).mockImplementation((_url, config) => { + const { page } = (config as { params: SpecimenListParams }).params + return Promise.resolve(specimenPage(page ?? 1, 120)) + }) + await render() + await screen.findByText('SUBJ-1') + await user.click(screen.getByRole('button', { name: 'Next' })) + await screen.findByText('SUBJ-51') + + await user.click(screen.getByRole('columnheader', { name: 'Collected' })) + await waitFor(() => + expect(lastSpecimenRequest()).toEqual( + expect.objectContaining({ page: 1, limit: 50, sort: 'collection_date', order: 'asc' }) + ) + ) + + await user.click(screen.getByRole('columnheader', { name: 'Collected' })) + await waitFor(() => + expect(lastSpecimenRequest()).toEqual( + expect.objectContaining({ page: 1, sort: 'collection_date', order: 'desc' }) + ) + ) + + await user.click(screen.getByRole('columnheader', { name: 'Created' })) + await waitFor(() => + expect(lastSpecimenRequest()).toEqual(expect.objectContaining({ sort: 'created', order: 'asc' })) + ) + }) + + it('shows the rows in the order the server returns them', async () => { + const user = userEvent.setup() + const page = specimenPage(1, 3, 3) + page.specimens[0].collectionDate = '2024-06-01' + page.specimens[1].collectionDate = '2022-01-01' + page.specimens[2].collectionDate = '2023-01-01' + vi.mocked(api.get).mockResolvedValue(page) + await render() + await screen.findByText('SUBJ-1') + + await user.click(screen.getByRole('columnheader', { name: 'Collected' })) + await waitFor(() => expect(specimenRequests()).toHaveLength(2)) + + const sources = screen + .getAllByRole('row') + .slice(1) + .map((row) => within(row).getAllByRole('cell')[1].textContent) + expect(sources).toEqual(['SUBJ-1', 'SUBJ-2', 'SUBJ-3']) + }) + + it('shows the skeleton, not the table or empty state, and disables Apply, while the first page loads', async () => { + vi.mocked(api.get).mockReturnValue(new Promise(() => {})) + + await render() + + expect(screen.getByRole('button', { name: /apply filters/i })).toBeDisabled() + expect(screen.queryByText('No specimens found')).not.toBeInTheDocument() + expect(screen.queryByRole('columnheader', { name: 'Collected' })).not.toBeInTheDocument() + expect(screen.queryByText(/Showing/)).not.toBeInTheDocument() + }) + + it('shows an inline error with retry when the page fails to load', async () => { + vi.mocked(api.get).mockRejectedValue(new Error('boom')) + + await render() + + expect(await screen.findByRole('button', { name: /try again/i })).toBeInTheDocument() + expect(screen.getByText('boom')).toBeInTheDocument() + }) +}) + describe('Specimens page URL filters', () => { beforeEach(() => { - vi.mocked(api.get).mockResolvedValue({ specimens: [] }) + vi.mocked(api.get).mockReset() + vi.mocked(api.get).mockResolvedValue(emptyPage) }) it('keeps filter edits local until Apply, and Back undoes the applied change', async () => { @@ -117,17 +258,34 @@ describe('Specimens page URL filters', () => { expect(page.sourceType()).toHaveValue('') }) - it('applies search as it is typed without adding a history entry per keystroke', async () => { + it('sends one search request after typing pauses, without a history entry per keystroke', async () => { + const user = userEvent.setup() const page = await renderAt('/start', '/specimens') + await waitFor(() => expect(specimenRequests()).toHaveLength(1)) - fireEvent.change(page.search(), { target: { value: 'S' } }) - fireEvent.change(page.search(), { target: { value: 'SU' } }) - await waitFor(() => expect(page.url()).toBe('/specimens?search=SU')) + await user.type(screen.getByRole('textbox', { name: /search specimens/i }), 'SUBJ') - fireEvent.click(screen.getByRole('button', { name: 'Back' })) + // Nothing applies while the user is still typing. + expect(page.url()).toBe('/specimens') + await waitFor(() => expect(page.url()).toBe('/specimens?search=SUBJ')) + await waitFor(() => expect(specimenRequests()).toHaveLength(2)) + await new Promise((resolve) => setTimeout(resolve, SPECIMEN_SEARCH_DEBOUNCE_MS * 2)) + expect(specimenRequests().map((params) => params.search)).toEqual([undefined, 'SUBJ']) + expect(specimenRequests()[1]).toEqual(expect.objectContaining({ page: 1, limit: 50 })) + + await user.click(screen.getByRole('button', { name: 'Back' })) await waitFor(() => expect(page.url()).toBe('/start')) }) + it('applies a typed search at once on Enter', async () => { + const user = userEvent.setup() + const page = await renderAt('/specimens') + + await user.type(screen.getByRole('textbox', { name: /search specimens/i }), 'QC{Enter}') + + expect(page.url()).toBe('/specimens?search=QC') + }) + it('keeps unapplied filter edits while a search is typed', async () => { const page = await renderAt('/specimens') From 52586509924085272bacc6a1ea64cb8b6ec7dbb2 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:47:46 -0700 Subject: [PATCH 28/54] fix(api): add lookup indexes and load bag and box papers in one query (#106) Bag and box detail ran one paper query per Sheet, and paper.sheet_id had no index, so each was a full scan of paper. Production files also lacked indexes that schema.ts declares, or held them under older names. - Bag and box detail load every paper of the parent in one query joined to sheet, then enrich them in one batch: a fixed number of queries whatever the Sheet count. Papers stay in id order, as the old full scan returned them; Sheet detail now orders by id too, since the new index would otherwise return them by sublabel. - Migration 011 adds tubes and static wells by Collection, cryovial tubes by barcode, lower(trim(barcode)) on both tube tables for the case-insensitive barcode check from #105, papers by Sheet and sublabel and by sublabel, Sheets by bag, and Container tags by Tag. Sheets by box already use the partial sheet_box_name_idx. - It also renames production's legacy-named indexes (idx_specimen_*, idx_location_new_*, ...) to the schema.ts names, drops the redundant idx_study_subject_study_name, and adds the schema.ts indexes production lacked. The unique Specimen collection event indexes are declared in schema.ts (new installs have them) but not added to production, whose data holds duplicate collection events. - schema-indexes.test.ts compares every schema.ts index (table, uniqueness, key columns or expressions, partial WHERE) with a new install, a version 1 install after the migrations, and a production-shaped file after the migrations, in both directions. - collection-detail-queries.test.ts checks the query count does not grow with Sheets, and EXPLAIN QUERY PLAN for paper-by-Sheet, tube-by-Collection and tube-by-barcode lookups. On a production snapshot, bag 166 detail went from 0.64 s to 18 ms, bag 442 from 1.18 s to 28 ms and box 2874 from 0.89 s to 22 ms, with identical responses. Co-Authored-By: Claude Opus 5.5 --- packages/api/initial_schema.sql | 12 +- packages/api/src/db/__tests__/client.test.ts | 12 +- .../src/db/__tests__/schema-evolution.test.ts | 29 ++ .../src/db/__tests__/schema-indexes.test.ts | 249 +++++++++++++++++ .../src/db/migrations/011_lookup_indexes.sql | 46 +++ packages/api/src/db/schema-evolution.ts | 2 +- packages/api/src/db/schema.ts | 25 +- .../collection-detail-queries.test.ts | 262 ++++++++++++++++++ .../src/lib/collections/collection-detail.ts | 41 ++- 9 files changed, 660 insertions(+), 18 deletions(-) create mode 100644 packages/api/src/db/__tests__/schema-indexes.test.ts create mode 100644 packages/api/src/db/migrations/011_lookup_indexes.sql create mode 100644 packages/api/src/lib/collections/__tests__/collection-detail-queries.test.ts diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index c132f42d..8b5d547f 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -175,6 +175,7 @@ CREATE TABLE IF NOT EXISTS storage_container_tag ( tag_id INTEGER NOT NULL REFERENCES tag(id), PRIMARY KEY (storage_container_id, tag_id) );--> statement-breakpoint +CREATE INDEX IF NOT EXISTS storage_container_tag_tag_id_idx ON storage_container_tag(tag_id);--> statement-breakpoint CREATE TABLE IF NOT EXISTS location ( id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES location(id), @@ -211,6 +212,8 @@ CREATE TABLE IF NOT EXISTS micronix_tube ( position TEXT );--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS micronix_tube_collection_position_idx ON micronix_tube(collection_id, position) WHERE position IS NOT NULL;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS micronix_tube_collection_id_idx ON micronix_tube(collection_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS micronix_tube_barcode_folded_idx ON micronix_tube(lower(trim(barcode)));--> statement-breakpoint CREATE TABLE IF NOT EXISTS cryovial_box ( id INTEGER PRIMARY KEY, location_id INTEGER NOT NULL REFERENCES location(id), @@ -229,6 +232,9 @@ CREATE TABLE IF NOT EXISTS cryovial_tube ( position TEXT );--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS cryovial_tube_collection_position_idx ON cryovial_tube(collection_id, position) WHERE position IS NOT NULL;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_collection_id_idx ON cryovial_tube(collection_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_barcode_idx ON cryovial_tube(barcode);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_barcode_folded_idx ON cryovial_tube(lower(trim(barcode)));--> statement-breakpoint CREATE TABLE IF NOT EXISTS box ( id INTEGER PRIMARY KEY, location_id INTEGER NOT NULL REFERENCES location(id), @@ -263,17 +269,21 @@ CREATE TABLE IF NOT EXISTS sheet ( (box_id IS NULL AND bag_id IS NULL)) );--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS sheet_box_name_idx ON sheet(box_id, name) WHERE box_id IS NOT NULL;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS sheet_bag_id_idx ON sheet(bag_id);--> statement-breakpoint CREATE TABLE IF NOT EXISTS paper ( id INTEGER PRIMARY KEY REFERENCES storage_container(id), sheet_id INTEGER NOT NULL REFERENCES sheet(id), sublabel TEXT );--> statement-breakpoint +CREATE INDEX IF NOT EXISTS paper_sheet_id_sublabel_idx ON paper(sheet_id, sublabel);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS paper_sublabel_idx ON paper(sublabel);--> statement-breakpoint CREATE TABLE IF NOT EXISTS static_well ( id INTEGER PRIMARY KEY REFERENCES storage_container(id), collection_id INTEGER NOT NULL REFERENCES micronix_plate(id), position TEXT );--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS static_well_collection_position_idx ON static_well(collection_id, position) WHERE position IS NOT NULL;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS static_well_collection_id_idx ON static_well(collection_id);--> statement-breakpoint CREATE TABLE IF NOT EXISTS strain ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, @@ -287,7 +297,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (10);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (11);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/client.test.ts b/packages/api/src/db/__tests__/client.test.ts index ba181e14..0fa9d243 100644 --- a/packages/api/src/db/__tests__/client.test.ts +++ b/packages/api/src/db/__tests__/client.test.ts @@ -27,14 +27,22 @@ describe('Database Client', () => { it('creates error_logs table when missing even if study and settings exist', () => { // Simulate database created before error_logs was added to schema const db = new Database(testDbPath) - db.exec('CREATE TABLE study (id INTEGER PRIMARY KEY)') + db.exec('CREATE TABLE study (id INTEGER PRIMARY KEY, title TEXT, short_code TEXT)') db.exec('CREATE TABLE settings (key TEXT, user_id INTEGER, value TEXT, PRIMARY KEY (key, user_id))') db.exec(`CREATE TABLE micronix_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE cryovial_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE static_well (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, position TEXT)`) db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL)`) db.exec(`CREATE TABLE sheet (id INTEGER PRIMARY KEY, name TEXT NOT NULL, box_id INTEGER, bag_id INTEGER)`) - db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, specimen_type_id INTEGER NOT NULL, collection_date DATE)`) + db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, study_subject_id INTEGER, control_batch_id INTEGER, specimen_type_id INTEGER NOT NULL, collection_date DATE)`) + // Tables migration 011 indexes + db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER)`) + db.exec(`CREATE TABLE storage_container_tag (storage_container_id INTEGER, tag_id INTEGER)`) + db.exec(`CREATE TABLE control_batch (id INTEGER PRIMARY KEY, control_definition_id INTEGER)`) + db.exec(`CREATE TABLE location (id INTEGER PRIMARY KEY, parent_id INTEGER, path TEXT)`) + db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER)`) + db.exec(`CREATE TABLE cryovial_box (id INTEGER PRIMARY KEY, location_id INTEGER)`) + db.exec(`CREATE TABLE bag (id INTEGER PRIMARY KEY, location_id INTEGER)`) // Intentionally omit paper — migration 003 preflight creates legacy stub // Intentionally do NOT create error_logs const before = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='error_logs'").get() diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index e531cc8b..4290d5dd 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -7,6 +7,7 @@ import { CURRENT_SCHEMA_VERSION, evolveOperationalSchema, getRecordedSchemaVersion, + hasSchemaVersionTable, isEmptyOperationalDatabase, } from '../schema-evolution' import { openOperationalDatabase } from '../open' @@ -54,6 +55,34 @@ function createTablesForLaterMigrations(sqlite: Database): void { user_id INTEGER NOT NULL REFERENCES users(id), expires_at INTEGER NOT NULL )`) + // Migration 011 indexes these columns. Some tests create a bare `study (id)` first. + ensureColumns(sqlite, 'study', ['title TEXT', 'short_code TEXT']) + ensureColumns(sqlite, 'specimen', ['control_batch_id INTEGER']) + ensureColumns(sqlite, 'storage_container', ['specimen_id INTEGER']) + ensureColumns(sqlite, 'storage_container_tag', ['storage_container_id INTEGER', 'tag_id INTEGER']) + ensureColumns(sqlite, 'control_batch', ['control_definition_id INTEGER']) + ensureColumns(sqlite, 'location', ['parent_id INTEGER', 'path TEXT']) + ensureColumns(sqlite, 'micronix_plate', ['location_id INTEGER']) + ensureColumns(sqlite, 'cryovial_box', ['location_id INTEGER']) + ensureColumns(sqlite, 'bag', ['location_id INTEGER']) + // Before version 3, migration 003 renames barcode to sublabel (and its preflight creates the + // table if missing), so only a database already past 003 gets paper in its later shape. + const version = hasSchemaVersionTable(sqlite) ? getRecordedSchemaVersion(sqlite) : null + if (version != null && version >= 3) ensureColumns(sqlite, 'paper', ['sheet_id INTEGER', 'sublabel TEXT']) +} + +/** Create `table` with `columns`, or add the ones an existing `table` lacks. */ +function ensureColumns(sqlite: Database, table: string, columns: string[]): void { + sqlite.exec(`CREATE TABLE IF NOT EXISTS ${table} (id INTEGER PRIMARY KEY)`) + const existing = new Set( + sqlite + .query<{ name: string }, []>(`SELECT name FROM pragma_table_info('${table}')`) + .all() + .map((row) => row.name), + ) + for (const column of columns) { + if (!existing.has(column.split(' ')[0])) sqlite.exec(`ALTER TABLE ${table} ADD COLUMN ${column}`) + } } describe('schema evolution', () => { diff --git a/packages/api/src/db/__tests__/schema-indexes.test.ts b/packages/api/src/db/__tests__/schema-indexes.test.ts new file mode 100644 index 00000000..073fb471 --- /dev/null +++ b/packages/api/src/db/__tests__/schema-indexes.test.ts @@ -0,0 +1,249 @@ +import { describe, it, expect } from 'vitest' +import { Database } from 'bun:sqlite' +import { readFileSync } from 'fs' +import { dirname, join } from 'path' +import { fileURLToPath } from 'url' +import { SQL, is } from 'drizzle-orm' +import { SQLiteSyncDialect, SQLiteTable, getTableConfig } from 'drizzle-orm/sqlite-core' +import * as schema from '../schema' +import { applyInitialSchema } from '../apply-initial-schema' +import { splitSqlStatements } from '../migration-runner' +import { CURRENT_SCHEMA_VERSION, evolveOperationalSchema, getRecordedSchemaVersion } from '../schema-evolution' + +/** + * Index definitions compared by name: table, uniqueness, key columns or expressions in order, + * and the partial-index WHERE. Text is normalised so that Drizzle's rendering + * (`lower(trim("micronix_tube"."barcode"))`) and SQLite's stored DDL + * (`lower(trim(barcode))`) compare equal. + */ +type IndexDefinition = { table: string; unique: boolean; keys: string[]; where: string | null } + +const dialect = new SQLiteSyncDialect() + +function normalise(text: string, table: string): string { + return text + .replace(/["`[\]]/g, '') + .replace(new RegExp(`\\b${table}\\.`, 'gi'), '') + .replace(/\s+/g, ' ') + .replace(/\s*([(),])\s*/g, '$1') + .trim() + .toLowerCase() +} + +/** Every index schema.ts declares, by name. Unique column constraints are not indexes here. */ +function declaredIndexes(): Map { + const declared = new Map() + for (const value of Object.values(schema)) { + if (!is(value, SQLiteTable)) continue + const { name: table, indexes } = getTableConfig(value) + for (const { config } of indexes) { + if (declared.has(config.name)) throw new Error(`index ${config.name} declared twice`) + declared.set(config.name, { + table, + unique: config.unique, + keys: config.columns.map((key) => normalise(is(key, SQL) ? dialect.sqlToQuery(key).sql : key.name, table)), + where: config.where ? normalise(dialect.sqlToQuery(config.where).sql, table) : null, + }) + } + } + return declared +} + +/** Split `a, f(b, c), d` on top-level commas. */ +function splitTopLevel(list: string): string[] { + const parts: string[] = [] + let depth = 0 + let start = 0 + for (let i = 0; i < list.length; i++) { + if (list[i] === '(') depth++ + else if (list[i] === ')') depth-- + else if (list[i] === ',' && depth === 0) { + parts.push(list.slice(start, i)) + start = i + 1 + } + } + parts.push(list.slice(start)) + return parts +} + +/** + * Every index a CREATE INDEX statement made, by name. Indexes SQLite creates for UNIQUE and + * PRIMARY KEY constraints have no SQL and are left out, as they are of declaredIndexes. + */ +function databaseIndexes(sqlite: Database): Map { + const rows = sqlite + .query<{ name: string; table: string; sql: string }, []>( + `SELECT name, tbl_name AS "table", sql FROM sqlite_master WHERE type = 'index' AND sql IS NOT NULL`, + ) + .all() + const found = new Map() + for (const { name, table, sql } of rows) { + const open = sql.indexOf('(', sql.search(/\bON\b/i)) + let depth = 0 + let close = open + for (; close < sql.length; close++) { + if (sql[close] === '(') depth++ + else if (sql[close] === ')' && --depth === 0) break + } + const where = sql.slice(close + 1).match(/^\s*WHERE\s+([\s\S]*?)\s*;?\s*$/i)?.[1] ?? null + const unique = sqlite + .query<{ unique: number }, []>(`SELECT "unique" FROM pragma_index_list('${table}') WHERE name = '${name}'`) + .get()!.unique + found.set(name, { + table, + unique: unique === 1, + keys: splitTopLevel(sql.slice(open + 1, close)).map((key) => normalise(key, table)), + where: where == null ? null : normalise(where, table), + }) + } + return found +} + +/** The differences between two index maps, one line per index, empty when they agree. */ +function indexDifferences(expected: Map, actual: Map): string[] { + const differences: string[] = [] + for (const [name, definition] of expected) { + const found = actual.get(name) + if (!found) differences.push(`missing ${name}`) + else if (JSON.stringify(found) !== JSON.stringify(definition)) { + differences.push(`${name}: expected ${JSON.stringify(definition)}, found ${JSON.stringify(found)}`) + } + } + for (const name of actual.keys()) { + if (!expected.has(name)) differences.push(`undeclared ${name}`) + } + return differences.sort() +} + +const VERSION_1_SNAPSHOT = join(dirname(fileURLToPath(import.meta.url)), 'fixtures', 'initial_schema_v1.sql') + +function openVersion1Install(): Database { + const sqlite = new Database(':memory:') + for (const statement of splitSqlStatements(readFileSync(VERSION_1_SNAPSHOT, 'utf-8'))) { + sqlite.exec(statement) + } + return sqlite +} + +/** + * Production files that predate the versioned snapshot hold some schema.ts indexes under other + * names, lack others, and have none of the indexes added later. Migration 011 renames the first + * and adds the second. Each entry: production name (null when production lacks the index), + * schema.ts name, and what the production index is on. + */ +const PRODUCTION_INDEX_NAMES: Array<[string | null, string, string]> = [ + ['idx_specimen_study_subject_id', 'specimen_study_subject_id_idx', 'specimen(study_subject_id)'], + ['idx_specimen_control_batch_id', 'specimen_control_batch_id_idx', 'specimen(control_batch_id)'], + ['idx_storage_container_specimen_id', 'storage_container_specimen_id_idx', 'storage_container(specimen_id)'], + ['idx_study_short_code', 'study_short_code_idx', 'study(short_code)'], + ['idx_study_subject_study_id', 'study_subject_study_id_idx', 'study_subject(study_id)'], + ['idx_study_subject_name', 'study_subject_name_idx', 'study_subject(name)'], + ['idx_location_new_parent_id', 'idx_location_parent_id', 'location(parent_id)'], + ['idx_location_new_path', 'idx_location_path', 'location(path)'], + [null, 'study_title_idx', ''], + [null, 'control_batch_control_definition_id_idx', ''], + [null, 'specimen_collection_date_idx', ''], + [null, 'specimen_specimen_type_id_idx', ''], + [null, 'micronix_plate_location_id_idx', ''], + [null, 'cryovial_box_location_id_idx', ''], + [null, 'bag_location_id_idx', ''], +] + +/** + * Indexes schema.ts declares that migrations leave off production files, because their data + * breaks them: those files hold more than one Specimen for some collection events. + */ +const NOT_ADDED_TO_PRODUCTION = ['idx_specimen_study_subject_type_date', 'idx_specimen_control_batch_type_date'] + +/** A version 1 install with production's index names, at the version the production snapshot had. */ +function openProductionShapedInstall(): Database { + const sqlite = openVersion1Install() + for (const [legacyName, schemaName, on] of PRODUCTION_INDEX_NAMES) { + sqlite.exec(`DROP INDEX ${schemaName}`) + if (legacyName) sqlite.exec(`CREATE INDEX ${legacyName} ON ${on}`) + } + for (const name of NOT_ADDED_TO_PRODUCTION) sqlite.exec(`DROP INDEX ${name}`) + // Redundant with the unique study_subject_study_name_idx that migration 005 adds. + sqlite.exec('CREATE INDEX idx_study_subject_study_name ON study_subject(study_id, name)') + return sqlite +} + +describe('indexes declared in schema.ts', () => { + const declared = declaredIndexes() + + it('finds the declarations, including expression and partial indexes', () => { + expect(declared.size).toBeGreaterThan(40) + expect(declared.get('micronix_tube_barcode_folded_idx')).toEqual({ + table: 'micronix_tube', + unique: false, + keys: ['lower(trim(barcode))'], + where: null, + }) + expect(declared.get('sheet_box_name_idx')).toEqual({ + table: 'sheet', + unique: true, + keys: ['box_id', 'name'], + where: 'box_id is not null', + }) + }) + + it('match a new install from initial_schema.sql', () => { + const sqlite = new Database(':memory:') + try { + applyInitialSchema(sqlite) + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(indexDifferences(declared, databaseIndexes(sqlite))).toEqual([]) + } finally { + sqlite.close() + } + }) + + it('match a version 1 install after the migrations', () => { + const sqlite = openVersion1Install() + try { + evolveOperationalSchema(sqlite) + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(indexDifferences(declared, databaseIndexes(sqlite))).toEqual([]) + } finally { + sqlite.close() + } + }) + + it('match a production file after the migrations, except the Specimen collection event indexes', () => { + const sqlite = openProductionShapedInstall() + try { + expect(indexDifferences(declared, databaseIndexes(sqlite))).not.toEqual([]) + evolveOperationalSchema(sqlite) + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(indexDifferences(declared, databaseIndexes(sqlite))).toEqual( + NOT_ADDED_TO_PRODUCTION.map((name) => `missing ${name}`).sort(), + ) + } finally { + sqlite.close() + } + }) + + it('notice a missing, undeclared or changed index', () => { + const sqlite = new Database(':memory:') + try { + applyInitialSchema(sqlite) + sqlite.exec('DROP INDEX paper_sublabel_idx') + sqlite.exec('CREATE INDEX paper_extra_idx ON paper(sublabel)') + sqlite.exec('DROP INDEX cryovial_tube_barcode_folded_idx') + sqlite.exec('CREATE INDEX cryovial_tube_barcode_folded_idx ON cryovial_tube(lower(barcode))') + sqlite.exec('DROP INDEX sheet_box_name_idx') + sqlite.exec('CREATE UNIQUE INDEX sheet_box_name_idx ON sheet(box_id, name)') + sqlite.exec('DROP INDEX static_well_collection_id_idx') + sqlite.exec('CREATE UNIQUE INDEX static_well_collection_id_idx ON static_well(collection_id)') + expect(indexDifferences(declared, databaseIndexes(sqlite)).map((line) => line.split(':')[0])).toEqual([ + 'cryovial_tube_barcode_folded_idx', + 'missing paper_sublabel_idx', + 'sheet_box_name_idx', + 'static_well_collection_id_idx', + 'undeclared paper_extra_idx', + ]) + } finally { + sqlite.close() + } + }) +}) diff --git a/packages/api/src/db/migrations/011_lookup_indexes.sql b/packages/api/src/db/migrations/011_lookup_indexes.sql new file mode 100644 index 00000000..73bf1a58 --- /dev/null +++ b/packages/api/src/db/migrations/011_lookup_indexes.sql @@ -0,0 +1,46 @@ +-- Migration 011: indexes for Collection, barcode, Sheet and Tag lookups, and the schema.ts +-- indexes that older production databases lack (issue #106). +-- +-- Production files that predate the versioned snapshot carry some schema.ts indexes under +-- other names (idx_specimen_study_subject_id for specimen_study_subject_id_idx, and so on). +-- Those are dropped and recreated under the schema.ts name, so every database has one index +-- per declaration. idx_study_subject_study_name duplicates the unique +-- study_subject_study_name_idx (migration 005) and is dropped. +-- +-- Not added here: the unique idx_specimen_study_subject_type_date and +-- idx_specimen_control_batch_type_date. New installs have them, but production files hold +-- duplicate collection events, and adding them would fail this migration. +DROP INDEX IF EXISTS idx_specimen_study_subject_id;--> statement-breakpoint +DROP INDEX IF EXISTS idx_specimen_control_batch_id;--> statement-breakpoint +DROP INDEX IF EXISTS idx_storage_container_specimen_id;--> statement-breakpoint +DROP INDEX IF EXISTS idx_study_short_code;--> statement-breakpoint +DROP INDEX IF EXISTS idx_study_subject_study_id;--> statement-breakpoint +DROP INDEX IF EXISTS idx_study_subject_name;--> statement-breakpoint +DROP INDEX IF EXISTS idx_study_subject_study_name;--> statement-breakpoint +DROP INDEX IF EXISTS idx_location_new_parent_id;--> statement-breakpoint +DROP INDEX IF EXISTS idx_location_new_path;--> statement-breakpoint +CREATE INDEX IF NOT EXISTS study_title_idx ON study(title);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS study_short_code_idx ON study(short_code);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS study_subject_study_id_idx ON study_subject(study_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS study_subject_name_idx ON study_subject(name);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS control_batch_control_definition_id_idx ON control_batch(control_definition_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS specimen_study_subject_id_idx ON specimen(study_subject_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS specimen_control_batch_id_idx ON specimen(control_batch_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS specimen_collection_date_idx ON specimen(collection_date);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS specimen_specimen_type_id_idx ON specimen(specimen_type_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS storage_container_specimen_id_idx ON storage_container(specimen_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS storage_container_tag_tag_id_idx ON storage_container_tag(tag_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS idx_location_parent_id ON location(parent_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS idx_location_path ON location(path);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS micronix_plate_location_id_idx ON micronix_plate(location_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS micronix_tube_collection_id_idx ON micronix_tube(collection_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS micronix_tube_barcode_folded_idx ON micronix_tube(lower(trim(barcode)));--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_box_location_id_idx ON cryovial_box(location_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_collection_id_idx ON cryovial_tube(collection_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_barcode_idx ON cryovial_tube(barcode);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS cryovial_tube_barcode_folded_idx ON cryovial_tube(lower(trim(barcode)));--> statement-breakpoint +CREATE INDEX IF NOT EXISTS bag_location_id_idx ON bag(location_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS sheet_bag_id_idx ON sheet(bag_id);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS paper_sheet_id_sublabel_idx ON paper(sheet_id, sublabel);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS paper_sublabel_idx ON paper(sublabel);--> statement-breakpoint +CREATE INDEX IF NOT EXISTS static_well_collection_id_idx ON static_well(collection_id); diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index a9283ee2..0cbaaf71 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -7,7 +7,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 10 +export const CURRENT_SCHEMA_VERSION = 11 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/db/schema.ts b/packages/api/src/db/schema.ts index 9ff59d80..0a4c2300 100644 --- a/packages/api/src/db/schema.ts +++ b/packages/api/src/db/schema.ts @@ -228,6 +228,14 @@ export const specimen = sqliteTable('specimen', { controlBatchIdIdx: index('specimen_control_batch_id_idx').on(table.controlBatchId), collectionDateIdx: index('specimen_collection_date_idx').on(table.collectionDate), specimenTypeIdIdx: index('specimen_specimen_type_id_idx').on(table.specimenTypeId), + // One collection event, one Specimen. Production databases that predate these indexes + // hold duplicates, so migrations do not add them there. + subjectTypeDateUniq: uniqueIndex('idx_specimen_study_subject_type_date') + .on(table.studySubjectId, table.specimenTypeId, table.collectionDate) + .where(sql`${table.studySubjectId} IS NOT NULL`), + controlBatchTypeDateUniq: uniqueIndex('idx_specimen_control_batch_type_date') + .on(table.controlBatchId, table.specimenTypeId, table.collectionDate) + .where(sql`${table.controlBatchId} IS NOT NULL`), })) export const storageContainer = sqliteTable('storage_container', { @@ -250,6 +258,7 @@ export const storageContainerTag = sqliteTable('storage_container_tag', { tagId: integer('tag_id').notNull().references(() => tag.id), }, (t) => ({ pk: primaryKey(t.storageContainerId, t.tagId), + tagIdIdx: index('storage_container_tag_tag_id_idx').on(t.tagId), })) // Locations (hierarchical parent-child structure) @@ -299,6 +308,10 @@ export const micronixTube = sqliteTable('micronix_tube', { collectionPositionUniq: uniqueIndex('micronix_tube_collection_position_idx') .on(table.collectionId, table.position) .where(sql`${table.position} IS NOT NULL`), + collectionIdIdx: index('micronix_tube_collection_id_idx').on(table.collectionId), + // Serves the case-insensitive barcode check (findContainersByBarcode with ignoreCase), + // which compares lower(trim(barcode)); SQLite uses the index only for that exact expression. + barcodeFoldedIdx: index('micronix_tube_barcode_folded_idx').on(sql`lower(trim(${table.barcode}))`), })) export const cryovialBox = sqliteTable('cryovial_box', { @@ -323,6 +336,10 @@ export const cryovialTube = sqliteTable('cryovial_tube', { collectionPositionUniq: uniqueIndex('cryovial_tube_collection_position_idx') .on(table.collectionId, table.position) .where(sql`${table.position} IS NOT NULL`), + collectionIdIdx: index('cryovial_tube_collection_id_idx').on(table.collectionId), + barcodeIdx: index('cryovial_tube_barcode_idx').on(table.barcode), + // See micronix_tube_barcode_folded_idx. + barcodeFoldedIdx: index('cryovial_tube_barcode_folded_idx').on(sql`lower(trim(${table.barcode}))`), })) export const box = sqliteTable('box', { @@ -364,13 +381,18 @@ export const sheet = sqliteTable('sheet', { `), unq: unique().on(t.name, t.boxId, t.bagId), boxNameUniq: uniqueIndex('sheet_box_name_idx').on(t.boxId, t.name).where(sql`${t.boxId} IS NOT NULL`), + // Sheets by box use sheet_box_name_idx: `box_id = ?` implies its `box_id IS NOT NULL`. + bagIdIdx: index('sheet_bag_id_idx').on(t.bagId), })) export const paper = sqliteTable('paper', { id: integer('id').primaryKey().references(() => storageContainer.id), sheetId: integer('sheet_id').notNull().references(() => sheet.id), sublabel: text('sublabel'), -}) +}, (table) => ({ + sheetIdSublabelIdx: index('paper_sheet_id_sublabel_idx').on(table.sheetId, table.sublabel), + sublabelIdx: index('paper_sublabel_idx').on(table.sublabel), +})) export const staticWell = sqliteTable('static_well', { id: integer('id').primaryKey().references(() => storageContainer.id), @@ -380,6 +402,7 @@ export const staticWell = sqliteTable('static_well', { collectionPositionUniq: uniqueIndex('static_well_collection_position_idx') .on(table.collectionId, table.position) .where(sql`${table.position} IS NOT NULL`), + collectionIdIdx: index('static_well_collection_id_idx').on(table.collectionId), })) // Additional reference tables diff --git a/packages/api/src/lib/collections/__tests__/collection-detail-queries.test.ts b/packages/api/src/lib/collections/__tests__/collection-detail-queries.test.ts new file mode 100644 index 00000000..024896ce --- /dev/null +++ b/packages/api/src/lib/collections/__tests__/collection-detail-queries.test.ts @@ -0,0 +1,262 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' +import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpers/db-setup' +import { + createTestBag, + createTestBox, + createTestCryovialBox, + createTestCryovialTube, + createTestLocation, + createTestMicronixPlate, + createTestMicronixTube, + createTestPaper, + createTestSheet, + createTestSpecimen, + createTestSpecimenType, + createTestStaticWell, + createTestStorageContainer, + createTestStorageType, + createTestUnit, +} from '../../../__tests__/helpers/factories' +import type { Database } from '../../../db/client' +import { findContainersByBarcode } from '../../identifier-resolution' +import { + getBagDetail, + getCryovialBoxDetail, + getGenericBoxDetail, + getMicronixPlateDetail, + getSheetDetail, +} from '../collection-detail' + +type Sqlite = Awaited>['sqlite'] + +describe('collection detail queries', () => { + let testDb: Database + let sqlite: Sqlite + let locationId: number + let containerOptions: { specimenId: number; unitId: number } + + beforeEach(async () => { + const setup = await setupTestDatabase() + testDb = setup.db + sqlite = setup.sqlite + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const loc = await createTestLocation(testDb, { + name: 'Freezer', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + locationId = loc.id + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'DBS' }) + const specimen = await createTestSpecimen(testDb, specimenType.id) + containerOptions = { specimenId: specimen.id, unitId: unit.id } + }) + + afterEach(() => { + vi.restoreAllMocks() + cleanupTestDatabase(sqlite) + }) + + /** Run `fn` and return the SQL of every statement it prepared. */ + async function capturePreparedSql(fn: () => Promise): Promise { + const statements: string[] = [] + const prepare = sqlite.prepare.bind(sqlite) + vi.spyOn(sqlite, 'prepare').mockImplementation(((query: string) => { + statements.push(query) + return prepare(query) + }) as typeof sqlite.prepare) + try { + await fn() + } finally { + vi.restoreAllMocks() + } + return statements + } + + /** EXPLAIN QUERY PLAN details for one statement, binding 1 to every parameter. */ + function queryPlan(statement: string): string[] { + const parameters = (statement.match(/\?/g) ?? []).map(() => 1) + return sqlite + .query<{ detail: string }, number[]>(`EXPLAIN QUERY PLAN ${statement}`) + .all(...parameters) + .map((row) => row.detail) + } + + /** Plans of the captured statements that read `table`, one per statement. */ + function plansReading(statements: string[], table: string): string[][] { + const reading = statements.filter((statement) => new RegExp(`\\bfrom "${table}"`, 'i').test(statement)) + expect(reading.length).toBeGreaterThan(0) + return reading.map(queryPlan) + } + + async function addPapers(sheetId: number, sublabels: Array) { + const ids: number[] = [] + for (const sublabel of sublabels) { + const container = await createTestStorageContainer(testDb, containerOptions) + await createTestPaper(testDb, { containerId: container.id, sheetId, sublabel }) + ids.push(container.id) + } + return ids + } + + /** A bag or box with `sheetCount` Sheets of two papers each. */ + async function addSheets(parent: { bagId: number } | { boxId: number }, sheetCount: number) { + for (let i = 1; i <= sheetCount; i++) { + const sheet = await createTestSheet(testDb, { name: `S${i}`, ...parent }) + await addPapers(sheet.id, [`S${i}-a`, `S${i}-b`]) + } + } + + it('runs the same number of queries for a bag with one Sheet as with eight', async () => { + const small = await createTestBag(testDb, { name: 'BAG-1', locationId }) + const large = await createTestBag(testDb, { name: 'BAG-8', locationId }) + await addSheets({ bagId: small.id }, 1) + await addSheets({ bagId: large.id }, 8) + + const smallQueries = await capturePreparedSql(() => getBagDetail(testDb, small.id)) + const largeQueries = await capturePreparedSql(() => getBagDetail(testDb, large.id)) + + expect(largeQueries.length).toBe(smallQueries.length) + }) + + it('runs the same number of queries for a box with one Sheet as with eight', async () => { + const small = await createTestBox(testDb, { name: 'BOX-1', locationId }) + const large = await createTestBox(testDb, { name: 'BOX-8', locationId }) + await addSheets({ boxId: small.id }, 1) + await addSheets({ boxId: large.id }, 8) + + const smallQueries = await capturePreparedSql(() => getGenericBoxDetail(testDb, small.id)) + const largeQueries = await capturePreparedSql(() => getGenericBoxDetail(testDb, large.id)) + + expect(largeQueries.length).toBe(smallQueries.length) + }) + + it('lists each paper under its own Sheet, keeps empty Sheets, and leaves out other bags', async () => { + const bag = await createTestBag(testDb, { name: 'BAG-A', locationId }) + const other = await createTestBag(testDb, { name: 'BAG-B', locationId }) + const first = await createTestSheet(testDb, { name: 'S1', bagId: bag.id }) + const empty = await createTestSheet(testDb, { name: 'S2', bagId: bag.id }) + const third = await createTestSheet(testDb, { name: 'S3', bagId: bag.id }) + const elsewhere = await createTestSheet(testDb, { name: 'S1', bagId: other.id }) + const [a, b] = await addPapers(first.id, ['a', 'b']) + const [c] = await addPapers(third.id, ['c']) + await addPapers(elsewhere.id, ['x']) + const [d] = await addPapers(first.id, [null]) + + const detail = await getBagDetail(testDb, bag.id) + + expect( + detail!.contents.sheets.map((sheet) => ({ + id: sheet.id, + papers: sheet.papers.map((p) => [p.id, p.sublabel, p.container?.id]), + })), + ).toEqual([ + { id: first.id, papers: [[a, 'a', a], [b, 'b', b], [d, null, d]] }, + { id: empty.id, papers: [] }, + { id: third.id, papers: [[c, 'c', c]] }, + ]) + }) + + it('lists a Sheet\'s papers in id order, not sublabel order', async () => { + const bag = await createTestBag(testDb, { name: 'BAG-O', locationId }) + const sheet = await createTestSheet(testDb, { name: 'S1', bagId: bag.id }) + const ids = await addPapers(sheet.id, ['b', 'a', null]) + + const sheetDetail = await getSheetDetail(testDb, sheet.id) + const bagDetail = await getBagDetail(testDb, bag.id) + + expect(sheetDetail!.papers.map((p) => p.id)).toEqual(ids) + expect(bagDetail!.contents.sheets[0].papers.map((p) => p.id)).toEqual(ids) + }) + + it('lists each paper under its own Sheet in a box', async () => { + const boxRecord = await createTestBox(testDb, { name: 'BOX-A', locationId }) + const first = await createTestSheet(testDb, { name: 'S1', boxId: boxRecord.id }) + const second = await createTestSheet(testDb, { name: 'S2', boxId: boxRecord.id }) + const [a] = await addPapers(second.id, ['a']) + const [b] = await addPapers(first.id, ['b']) + + const detail = await getGenericBoxDetail(testDb, boxRecord.id) + + expect(detail!.contents.sheets.map((sheet) => [sheet.id, sheet.papers.map((p) => p.id)])).toEqual([ + [first.id, [b]], + [second.id, [a]], + ]) + }) + + describe('use an index', () => { + it('to find papers by Sheet in bag, box and Sheet detail', async () => { + const bag = await createTestBag(testDb, { name: 'BAG-P', locationId }) + const boxRecord = await createTestBox(testDb, { name: 'BOX-P', locationId }) + const sheet = await createTestSheet(testDb, { name: 'S1', bagId: bag.id }) + await createTestSheet(testDb, { name: 'S1', boxId: boxRecord.id }) + await addPapers(sheet.id, ['a']) + + const statements = [ + ...(await capturePreparedSql(() => getBagDetail(testDb, bag.id))), + ...(await capturePreparedSql(() => getGenericBoxDetail(testDb, boxRecord.id))), + ...(await capturePreparedSql(() => getSheetDetail(testDb, sheet.id))), + ] + // Papers by Sheet or by the Sheet's parent. The Container read view also reads paper, by id. + const bySheet = statements.filter( + (statement) => + /\bfrom "paper"/i.test(statement) && /"(paper"\."sheet_id|sheet"\."bag_id|sheet"\."box_id)" = \?/.test(statement), + ) + expect(bySheet).toHaveLength(3) + const plans = bySheet.map(queryPlan) + for (const plan of plans) { + expect(plan.join('\n')).toMatch(/SEARCH p(aper)? USING (COVERING )?INDEX paper_sheet_id_sublabel_idx \(sheet_id=\?\)/) + expect(plan.join('\n')).not.toMatch(/SCAN (paper|sheet)\b/) + } + }) + + it('to find tubes and static wells by Collection', async () => { + const plate = await createTestMicronixPlate(testDb, { name: 'P1', locationId }) + const cryovialBox = await createTestCryovialBox(testDb, { name: 'C1', locationId }) + const tube = await createTestStorageContainer(testDb, containerOptions) + await createTestMicronixTube(testDb, { containerId: tube.id, collectionId: plate.id, barcode: 'MX1', position: 'A01' }) + const well = await createTestStorageContainer(testDb, containerOptions) + await createTestStaticWell(testDb, { containerId: well.id, collectionId: plate.id, position: 'A02' }) + const vial = await createTestStorageContainer(testDb, containerOptions) + await createTestCryovialTube(testDb, { containerId: vial.id, collectionId: cryovialBox.id, barcode: 'CV1', position: 'A01' }) + + const statements = [ + ...(await capturePreparedSql(() => getMicronixPlateDetail(testDb, plate.id))), + ...(await capturePreparedSql(() => getCryovialBoxDetail(testDb, cryovialBox.id))), + ] + const collectionLookups = (table: string) => + plansReading(statements, table).filter((plan) => plan.some((line) => line.includes('collection_id=?'))) + expect(collectionLookups('micronix_tube')).toEqual([ + ['SEARCH micronix_tube USING INDEX micronix_tube_collection_id_idx (collection_id=?)'], + ]) + expect(collectionLookups('static_well')).toEqual([ + ['SEARCH static_well USING INDEX static_well_collection_id_idx (collection_id=?)'], + ]) + expect(collectionLookups('cryovial_tube')).toEqual([ + ['SEARCH cryovial_tube USING INDEX cryovial_tube_collection_id_idx (collection_id=?)'], + ]) + }) + + it('to find tubes by barcode, exactly and ignoring case and surrounding whitespace', async () => { + const exact = await capturePreparedSql(() => findContainersByBarcode(testDb, 'MX1')) + const folded = await capturePreparedSql(() => + findContainersByBarcode(testDb, ' mx1 ', { ignoreCase: true, tubesOnly: true }), + ) + + const lookup = (plans: string[][]) => plans.map((plan) => plan.find((line) => /^(SEARCH|SCAN) (micronix_tube|cryovial_tube|paper)\b/.test(line))) + expect(lookup(plansReading(exact, 'micronix_tube'))).toEqual([ + 'SEARCH micronix_tube USING INDEX sqlite_autoindex_micronix_tube_1 (barcode=?)', + ]) + expect(lookup(plansReading(exact, 'cryovial_tube'))).toEqual([ + 'SEARCH cryovial_tube USING INDEX cryovial_tube_barcode_idx (barcode=?)', + ]) + expect(lookup(plansReading(exact, 'paper'))).toEqual(['SEARCH paper USING INDEX paper_sublabel_idx (sublabel=?)']) + expect(lookup(plansReading(folded, 'micronix_tube'))).toEqual([ + 'SEARCH micronix_tube USING INDEX micronix_tube_barcode_folded_idx (=?)', + ]) + expect(lookup(plansReading(folded, 'cryovial_tube'))).toEqual([ + 'SEARCH cryovial_tube USING INDEX cryovial_tube_barcode_folded_idx (=?)', + ]) + }) + }) +}) diff --git a/packages/api/src/lib/collections/collection-detail.ts b/packages/api/src/lib/collections/collection-detail.ts index 330c7164..4250a403 100644 --- a/packages/api/src/lib/collections/collection-detail.ts +++ b/packages/api/src/lib/collections/collection-detail.ts @@ -120,14 +120,30 @@ export async function getCryovialBoxDetail(database: Database, id: number) { } } -async function getSheetContentsForParent(database: Database, sheets: (typeof sheet.$inferSelect)[]) { - return Promise.all( - sheets.map(async (s) => { - const papers = await database.select().from(paper).where(eq(paper.sheetId, s.id)) - const paperEntries = await enrichPaperContainers(database, papers) - return { ...s, papers: paperEntries } - }), - ) +/** + * The Sheets of one box or bag, each with its papers. Papers load in one query joined to + * their Sheet's parent, so the number of queries does not grow with the number of Sheets. + */ +async function getSheetContentsForParent(database: Database, parent: { boxId: number } | { bagId: number }) { + const parentMatches = 'boxId' in parent ? eq(sheet.boxId, parent.boxId) : eq(sheet.bagId, parent.bagId) + const [sheets, papers] = await Promise.all([ + database.select().from(sheet).where(parentMatches), + database + .select({ id: paper.id, sheetId: paper.sheetId, sublabel: paper.sublabel }) + .from(paper) + .innerJoin(sheet, eq(sheet.id, paper.sheetId)) + .where(parentMatches) + .orderBy(paper.id), + ]) + + const paperEntries = await enrichPaperContainers(database, papers) + const entriesBySheetId = new Map() + papers.forEach((p, i) => { + const entries = entriesBySheetId.get(p.sheetId) ?? [] + entries.push(paperEntries[i]) + entriesBySheetId.set(p.sheetId, entries) + }) + return sheets.map((s) => ({ ...s, papers: entriesBySheetId.get(s.id) ?? [] })) } export async function getGenericBoxDetail(database: Database, id: number) { @@ -135,8 +151,7 @@ export async function getGenericBoxDetail(database: Database, id: number) { if (!boxRecord) return null const loc = await database.select().from(location).where(eq(location.id, boxRecord.locationId)).get() - const sheets = await database.select().from(sheet).where(eq(sheet.boxId, id)) - const sheetContents = await getSheetContentsForParent(database, sheets) + const sheetContents = await getSheetContentsForParent(database, { boxId: id }) return { box: { @@ -153,8 +168,7 @@ export async function getBagDetail(database: Database, id: number) { if (!bagRecord) return null const loc = await database.select().from(location).where(eq(location.id, bagRecord.locationId)).get() - const sheets = await database.select().from(sheet).where(eq(sheet.bagId, id)) - const contents = await getSheetContentsForParent(database, sheets) + const contents = await getSheetContentsForParent(database, { bagId: id }) return { bag: { @@ -197,7 +211,8 @@ export async function getSheetDetail(database: Database, id: number) { parentBag = parent?.bag ? { id: parent.bag.id, name: parent.bag.name } : null } - const papers = await database.select().from(paper).where(eq(paper.sheetId, id)) + // By id: paper_sheet_id_sublabel_idx would otherwise return them in sublabel order. + const papers = await database.select().from(paper).where(eq(paper.sheetId, id)).orderBy(paper.id) const paperEntries = await enrichPaperContainers(database, papers) return { From aabf5d8368d79d14a21bc3d89b504b693c485caf Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:48:44 -0700 Subject: [PATCH 29/54] fix(api): return 4xx for bad input and honour location_id on the containers list Closes #120. - Shared query-param helpers (src/lib/query-params.ts): a page above 1,000,000 is a 400 on every list (it reached SQLite as a float offset and failed with "datatype mismatch"); a non-numeric id filter (subject_id, control_batch_id, specimen_type_id, specimen_id, location_id, tag_ids, export subject_ids/specimen_type_ids/tag_ids) is a 400 naming the parameter instead of being dropped. - GET /containers?location_id filters to Containers at that Location or under it, through containerAtLocation from container-selection (ADR 0011). - Container export: a malformed or non-array columns value is a 400. - PATCH /auth/me, PUT /auth/users/:id and PATCH Derivation with an empty body are a 400 instead of Drizzle's "No values to set". - assertUserIdentityFree: an email or username held by a soft-deleted user is a 409 telling the admin to restore that user (self-register, register, /me, admin update); /me, admin update and restore now return their errors through handleRouteError. Restore also checks the username. - PATCH /containers/:id rejects an unknown unit id with a 400, including on a Container whose type is unknown (was an FK 409/500). The statistics test that pinned "ignores a non-numeric id" now asserts the 400 the issue asks for. Co-Authored-By: Claude Opus 5.5 --- CODING_STANDARDS.md | 2 +- packages/api/src/lib/auth/auth-service.ts | 47 ++++-- packages/api/src/lib/common-validators.ts | 8 + packages/api/src/lib/constants.ts | 16 +- .../src/lib/container-selection/conditions.ts | 5 +- .../api/src/lib/container-selection/index.ts | 2 +- packages/api/src/lib/query-params.ts | 53 +++++++ .../api/src/lib/specimens/specimen-read.ts | 25 ++-- .../api/src/lib/statistics/parse-filters.ts | 19 +-- .../api/src/routes/__tests__/auth.test.ts | 118 +++++++++++++++ .../src/routes/__tests__/containers.test.ts | 73 +++++++++ .../src/routes/__tests__/derivations.test.ts | 10 ++ .../src/routes/__tests__/error-logs.test.ts | 10 ++ .../api/src/routes/__tests__/export.test.ts | 32 ++++ .../src/routes/__tests__/locations.test.ts | 24 +++ .../src/routes/__tests__/specimens.test.ts | 34 +++++ .../src/routes/__tests__/statistics.test.ts | 13 +- .../api/src/routes/__tests__/studies.test.ts | 10 ++ .../api/src/routes/__tests__/subjects.test.ts | 10 ++ packages/api/src/routes/auth.ts | 139 ++++-------------- packages/api/src/routes/containers.ts | 27 ++-- packages/api/src/routes/derivations.ts | 7 +- packages/api/src/routes/error-logs.ts | 3 +- packages/api/src/routes/export.ts | 39 +++-- packages/api/src/routes/locations.ts | 8 +- 25 files changed, 528 insertions(+), 206 deletions(-) create mode 100644 packages/api/src/lib/query-params.ts diff --git a/CODING_STANDARDS.md b/CODING_STANDARDS.md index f0f168c8..e20cd87f 100644 --- a/CODING_STANDARDS.md +++ b/CODING_STANDARDS.md @@ -32,7 +32,7 @@ Paths are relative to `packages/api/`. - Return the full `SpecimenSource` shape to every caller, not a per-endpoint subset (ADR 0007). - Load a Container's read view (placement, identity, Tags, unit, location, Specimen, Source) with `loadContainerReadViews` or `loadContainerReadViewsByIds` from `src/lib/container-read-view.ts`, not with per-endpoint queries (ADR 0009). - Map the read view to its read path's type: `EnrichedContainerApi` for list, derivations and PATCH, `EnrichedStorageContainer` for collection detail, and CSV row fields for Container export (ADR 0009). -- To choose which Containers an export or statistic covers, call `selectContainers` from `src/lib/container-selection/`, and filter a paginated Container query by Tags with `containerHasAllTags` (ADR 0011). +- To choose which Containers an export or statistic covers, call `selectContainers` from `src/lib/container-selection/`, and filter a paginated Container query by Tags with `containerHasAllTags` and by Location with `containerAtLocation` (ADR 0011). ### Routes diff --git a/packages/api/src/lib/auth/auth-service.ts b/packages/api/src/lib/auth/auth-service.ts index 7b4663d7..12d0b4c8 100644 --- a/packages/api/src/lib/auth/auth-service.ts +++ b/packages/api/src/lib/auth/auth-service.ts @@ -5,7 +5,7 @@ import { nanoid } from 'nanoid' import type { Database } from '../../db/client' import { users, sessions, type users as usersTable } from '../../db/schema' import { getSessionSettings } from '../settings' -import { UnauthorizedError, ValidationError } from '../error-handler' +import { ConflictError, UnauthorizedError, ValidationError } from '../error-handler' import { utcNow } from '../datetime' export type PublicUser = { @@ -151,6 +151,36 @@ export async function deleteUserSessions( : eq(sessions.userId, userId)) } +/** + * Check that an email and a username can be given to a user: `userId`, or a new user when + * omitted. Emails and usernames are unique across every user, soft-deleted ones included, so + * a deleted user still holds theirs. An active holder throws a `ValidationError` (400, as + * before); a soft-deleted holder throws a `ConflictError` (409) that says to restore that user. + */ +export async function assertUserIdentityFree( + database: Database, + identity: { email?: string | null; username?: string | null }, + userId?: number +): Promise { + const fields = [ + { label: 'Email', column: users.email, value: identity.email }, + { label: 'Username', column: users.username, value: identity.username }, + ] + for (const { label, column, value } of fields) { + if (!value) continue + const holder = await database + .select({ id: users.id, deletedAt: users.deletedAt }) + .from(users) + .where(userId === undefined ? eq(column, value) : and(eq(column, value), ne(users.id, userId))) + .get() + if (!holder) continue + if (holder.deletedAt === null) throw new ValidationError(`${label} already in use`) + throw new ConflictError( + `${label} already in use by a deleted user. Restore that user, or choose another ${label.toLowerCase()}.` + ) + } +} + export async function registerApprovedUser( database: Database, data: { @@ -161,20 +191,7 @@ export async function registerApprovedUser( role: string } ): Promise { - const existingEmail = await database.select().from(users).where(eq(users.email, data.email)).get() - if (existingEmail) { - throw new ValidationError('Email already in use') - } - if (data.username) { - const existingUsername = await database - .select() - .from(users) - .where(eq(users.username, data.username)) - .get() - if (existingUsername) { - throw new ValidationError('Username already in use') - } - } + await assertUserIdentityFree(database, { email: data.email, username: data.username }) const passwordHash = await bcrypt.hash(data.password, BCRYPT_COST) const createdAt = utcNow() const [user] = await database diff --git a/packages/api/src/lib/common-validators.ts b/packages/api/src/lib/common-validators.ts index b2cab1dd..08059a12 100644 --- a/packages/api/src/lib/common-validators.ts +++ b/packages/api/src/lib/common-validators.ts @@ -119,3 +119,11 @@ export function validateId(id: number | null | undefined): { valid: boolean; err } return { valid: true } } + +/** + * True when a parsed PATCH or PUT body sets no field. Drizzle throws "No values to set" on an + * empty `.set({})`, so a route rejects such a body with a 400 before it updates. + */ +export function setsNoField(data: Record): boolean { + return Object.values(data).every((value) => value === undefined) +} diff --git a/packages/api/src/lib/constants.ts b/packages/api/src/lib/constants.ts index ceaef9fb..b32f95f7 100644 --- a/packages/api/src/lib/constants.ts +++ b/packages/api/src/lib/constants.ts @@ -1,4 +1,5 @@ import type { Database } from '../db/client' +import { pageQueryParam } from './query-params' // Cache for pagination settings let paginationCache: { defaultPageSize: number; maxPageSize: number } | null = null @@ -43,16 +44,9 @@ export async function validateLimit(db: Database, limit: string | number | undef } /** - * Validates and normalizes a page number - * @param page - The page number to validate (can be string or number) - * @returns A valid page number (minimum 1) + * Validates and normalizes a page number: page 1 when omitted or not a number, at least 1, and + * a `ValidationError` (400) above `MAX_PAGE`. `name` is the query parameter, for the message. */ -export function validatePage(page: string | number | undefined): number { - const parsed = typeof page === 'string' ? parseInt(page, 10) : page - if (isNaN(parsed as number) || parsed === undefined) { - return 1 - } - const num = Number(parsed) - return Math.max(1, num) +export function validatePage(page: string | number | undefined, name = 'page'): number { + return pageQueryParam(page, name) } - diff --git a/packages/api/src/lib/container-selection/conditions.ts b/packages/api/src/lib/container-selection/conditions.ts index 1e66d99e..c92cbfb7 100644 --- a/packages/api/src/lib/container-selection/conditions.ts +++ b/packages/api/src/lib/container-selection/conditions.ts @@ -81,9 +81,10 @@ export function containerFilterConditions(filters: ContainerSelectionFilters): S /** * Containers stored in a Location or any Location under it: tubes and static wells on - * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags there. + * micronix plates, tubes in cryovial boxes, and papers on sheets in boxes or bags there. A + * condition on `storage_container`, so a caller can page and count in SQL. */ -function containerAtLocation(locationId: number): SQL { +export function containerAtLocation(locationId: number): SQL { const here = sql`SELECT id FROM selected_location` return sql`${storageContainer.id} IN ( WITH RECURSIVE selected_location(id) AS ( diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index 7be0fbcd..a8310a64 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -26,7 +26,7 @@ import type { } from './types' export * from './types' -export { containerHasAllTags } from './conditions' +export { containerAtLocation, containerHasAllTags } from './conditions' export { containerTypesFilter, specimenSourceType } from './filters' /** Select the Containers and Specimens a selection matches, with each Container's type. */ diff --git a/packages/api/src/lib/query-params.ts b/packages/api/src/lib/query-params.ts new file mode 100644 index 00000000..0bf4d4bc --- /dev/null +++ b/packages/api/src/lib/query-params.ts @@ -0,0 +1,53 @@ +/** + * Numeric query parameters (issue #120). A list filter such as `subject_id=abc` used to be + * dropped, so the list returned every row, and an oversized `page` reached SQLite as a float + * offset and failed with a 500. These helpers reject such values with a `ValidationError` (400) + * that names the parameter. + */ +import { ValidationError } from './error-handler' + +/** + * The highest page number a list accepts. Far above any real list, and low enough that + * `(page - 1) * limit` stays a safe integer for any page size. + */ +export const MAX_PAGE = 1_000_000 + +const POSITIVE_INTEGER = /^\d+$/ + +function positiveId(name: string, value: string): number { + const text = value.trim() + const id = POSITIVE_INTEGER.test(text) ? Number(text) : NaN + if (!Number.isSafeInteger(id) || id < 1) { + throw new ValidationError(`Invalid ${name} '${value}': use a positive whole number.`) + } + return id +} + +/** An id filter. Empty or omitted means no filter; anything but a positive integer throws. */ +export function idQueryParam(name: string, value: string | null | undefined): number | undefined { + if (value == null || value.trim() === '') return undefined + return positiveId(name, value) +} + +/** + * A repeated id filter (`tag_ids=1&tag_ids=2`). Omitted or all-empty means no filter; any value + * that is not a positive integer throws. + */ +export function idListQueryParam(name: string, values: readonly string[] | null | undefined): number[] | undefined { + const present = (values ?? []).filter((value) => value.trim() !== '') + if (present.length === 0) return undefined + return present.map((value) => positiveId(name, value)) +} + +/** + * A page number. Omitted, empty or below 1 means page 1, and text that does not start with a + * number does too, as before; a page above `MAX_PAGE` throws. + */ +export function pageQueryParam(value: string | number | undefined, name = 'page'): number { + const parsed = typeof value === 'string' ? parseInt(value, 10) : value + if (parsed === undefined || Number.isNaN(parsed)) return 1 + if (parsed > MAX_PAGE) { + throw new ValidationError(`Invalid ${name} '${value}': the highest page is ${MAX_PAGE}.`) + } + return Math.max(1, Math.trunc(parsed)) +} diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 01c7d52e..82604875 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -2,6 +2,7 @@ import type { Database } from '../../db/client' import { specimen, storageContainer, studySubject, study, specimenType, controlBatch } from '../../db/schema' import { eq, and, or, sql, inArray } from 'drizzle-orm' import { validatePage, validateLimit } from '../constants' +import { idQueryParam } from '../query-params' import { findContainersByBarcode } from '../identifier-resolution' import { collectionDateFilter, timestampFilter } from '../date-filters' import { dateRangeConditions } from '../sql-helpers' @@ -122,25 +123,19 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer conditions.push(eq(study.shortCode, studyCode)) } - if (subjectId) { - const id = parseInt(subjectId) - if (!isNaN(id)) { - conditions.push(eq(specimen.studySubjectId, id)) - } + const subjectIdFilter = idQueryParam('subject_id', subjectId) + if (subjectIdFilter !== undefined) { + conditions.push(eq(specimen.studySubjectId, subjectIdFilter)) } - if (controlBatchId) { - const id = parseInt(controlBatchId) - if (!isNaN(id)) { - conditions.push(eq(specimen.controlBatchId, id)) - } + const controlBatchIdFilter = idQueryParam('control_batch_id', controlBatchId) + if (controlBatchIdFilter !== undefined) { + conditions.push(eq(specimen.controlBatchId, controlBatchIdFilter)) } - if (specimenTypeId) { - const id = parseInt(specimenTypeId) - if (!isNaN(id)) { - conditions.push(eq(specimen.specimenTypeId, id)) - } + const specimenTypeIdFilter = idQueryParam('specimen_type_id', specimenTypeId) + if (specimenTypeIdFilter !== undefined) { + conditions.push(eq(specimen.specimenTypeId, specimenTypeIdFilter)) } const collectionFrom = collectionDateFilter('collection_date_from', collectionDateFrom) diff --git a/packages/api/src/lib/statistics/parse-filters.ts b/packages/api/src/lib/statistics/parse-filters.ts index 9210fe79..b833e86f 100644 --- a/packages/api/src/lib/statistics/parse-filters.ts +++ b/packages/api/src/lib/statistics/parse-filters.ts @@ -1,29 +1,22 @@ import type { Context } from 'hono' import { collectionDateFilter, timestampFilter } from '../date-filters' +import { idListQueryParam, idQueryParam } from '../query-params' import type { StatisticsFilters } from './types' -/** Parse an integer id query value; anything else means no filter. */ -function parseId(value: string | undefined): number | undefined { - if (!value) return undefined - const id = parseInt(value) - return isNaN(id) ? undefined : id -} - /** - * Parse dashboard statistics query parameters from a Hono request. A malformed date bound - * throws a `ValidationError` (400). + * Parse dashboard statistics query parameters from a Hono request. A malformed date bound or + * a non-numeric id throws a `ValidationError` (400). */ export function parseStatisticsFilters(c: Context): StatisticsFilters { - const tagIdsParam = c.req.queries('tag_ids') || c.req.queries('tag_ids[]') - const tagIds = tagIdsParam?.map((id) => parseInt(id)).filter((id) => !isNaN(id)) + const tagIds = idListQueryParam('tag_ids', c.req.queries('tag_ids') || c.req.queries('tag_ids[]')) return { study: c.req.query('study'), source_type: c.req.query('source_type'), - specimen_type_id: parseId(c.req.query('specimen_type_id')), + specimen_type_id: idQueryParam('specimen_type_id', c.req.query('specimen_type_id')), container_type: c.req.query('container_type'), tag_ids: tagIds, - location_id: parseId(c.req.query('location_id')), + location_id: idQueryParam('location_id', c.req.query('location_id')), collection_date_from: collectionDateFilter('collection_date_from', c.req.query('collection_date_from')), collection_date_to: collectionDateFilter('collection_date_to', c.req.query('collection_date_to')), created_from: timestampFilter('created_from', c.req.query('created_from')), diff --git a/packages/api/src/routes/__tests__/auth.test.ts b/packages/api/src/routes/__tests__/auth.test.ts index cffc1f76..7ee9e8f9 100644 --- a/packages/api/src/routes/__tests__/auth.test.ts +++ b/packages/api/src/routes/__tests__/auth.test.ts @@ -903,6 +903,124 @@ describe('Auth API', () => { expect(data.user.role).toBe('member') }) }) + describe('bad input returns 4xx (#120)', () => { + let deletedUser: typeof users.$inferSelect + let member: typeof users.$inferSelect + let memberCookie: string + + beforeEach(async () => { + deletedUser = await createTestUser(testDb, { + email: 'gone@example.com', + name: 'Gone User', + username: 'goneuser', + password: 'password123', + }) + await testDb.update(users).set({ deletedAt: utcNow() }).where(eq(users.id, deletedUser.id)) + member = await createTestUser(testDb, { + email: 'member@example.com', + name: 'Member', + username: 'memberuser', + password: 'password123', + }) + memberCookie = await loginAndGetCookie(app, 'member@example.com', 'password123') + }) + + async function expectDeletedHolderConflict(res: Response, field: 'Email' | 'Username') { + expect(res.status).toBe(409) + const data = await res.json() as ErrorResponse + expect(data.error).toBe( + `${field} already in use by a deleted user. Restore that user, or choose another ${field.toLowerCase()}.` + ) + } + + it('returns 400 for an empty PATCH /me body and leaves the user unchanged', async () => { + const res = await authenticatedRequest(app, '/api/auth/me', { method: 'PATCH', cookie: memberCookie, json: {} }) + expect(res.status).toBe(400) + expect((await res.json() as ErrorResponse).error).toBe('Nothing to update: send at least one field to change.') + const row = await testDb.select().from(users).where(eq(users.id, member.id)).get() + expect(row?.name).toBe('Member') + }) + + it('returns 400 for an empty PUT /users/:id body', async () => { + const res = await authenticatedRequest(app, `/api/auth/users/${member.id}`, { + method: 'PUT', + cookie: adminCookieHeader, + json: {}, + }) + expect(res.status).toBe(400) + expect((await res.json() as ErrorResponse).error).toBe('Nothing to update: send at least one field to change.') + }) + + it.each([ + ['Email', { email: 'gone@example.com' }], + ['Username', { username: 'goneuser' }], + ] as const)('returns 409 when PATCH /me takes the %s of a deleted user', async (field, json) => { + const res = await authenticatedRequest(app, '/api/auth/me', { method: 'PATCH', cookie: memberCookie, json }) + await expectDeletedHolderConflict(res, field) + }) + + it.each([ + ['Email', { email: 'gone@example.com' }], + ['Username', { username: 'goneuser' }], + ] as const)('returns 409 when an admin gives a user the %s of a deleted user', async (field, json) => { + const res = await authenticatedRequest(app, `/api/auth/users/${member.id}`, { + method: 'PUT', + cookie: adminCookieHeader, + json, + }) + await expectDeletedHolderConflict(res, field) + }) + + it.each([ + ['Email', { email: 'gone@example.com', username: 'fresh' }], + ['Username', { email: 'fresh@example.com', username: 'goneuser' }], + ] as const)('returns 409 when self-registering with the %s of a deleted user', async (field, identity) => { + const res = await app.request('/api/auth/self-register', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ ...identity, name: 'New', password: 'password123' }), + }) + await expectDeletedHolderConflict(res, field) + }) + + it('returns 409 when an admin registers a user with the email of a deleted user', async () => { + const res = await authenticatedRequest(app, '/api/auth/register', { + method: 'POST', + cookie: adminCookieHeader, + json: { email: 'gone@example.com', name: 'New', password: 'password123', role: 'member' }, + }) + await expectDeletedHolderConflict(res, 'Email') + }) + + it('still returns 400 when an active user holds the email', async () => { + const res = await authenticatedRequest(app, '/api/auth/me', { + method: 'PATCH', + cookie: memberCookie, + json: { email: 'admin@test.com' }, + }) + expect(res.status).toBe(400) + expect((await res.json() as ErrorResponse).error).toBe('Email already in use') + }) + + it('lets a user keep their own email and username', async () => { + const res = await authenticatedRequest(app, '/api/auth/me', { + method: 'PATCH', + cookie: memberCookie, + json: { email: 'member@example.com', username: 'memberuser', name: 'Renamed' }, + }) + expect(res.status).toBe(200) + }) + + it('restores a deleted user whose email and username are free', async () => { + const res = await authenticatedRequest(app, `/api/auth/users/${deletedUser.id}/restore`, { + method: 'POST', + cookie: adminCookieHeader, + }) + expect(res.status).toBe(200) + const row = await testDb.select().from(users).where(eq(users.id, deletedUser.id)).get() + expect(row?.deletedAt).toBeNull() + }) + }) }) diff --git a/packages/api/src/routes/__tests__/containers.test.ts b/packages/api/src/routes/__tests__/containers.test.ts index b1e35dec..a5ee4609 100644 --- a/packages/api/src/routes/__tests__/containers.test.ts +++ b/packages/api/src/routes/__tests__/containers.test.ts @@ -149,6 +149,67 @@ describe('Containers API', () => { }) }) + describe('GET /api/containers bad input (#120)', () => { + it.each([ + ['page', '99999999999999999999'], + ['specimen_id', 'abc'], + ['location_id', 'abc'], + ['tag_ids', '1x'], + ])('returns 400 for %s=%s', async (param, value) => { + const res = await ctx.request(`/api/containers?${param}=${value}`) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + + it('returns 400 when one of several tag_ids is not a number', async () => { + const tagId = (await createTestTag(ctx.db, { name: 'red' })).id + const res = await ctx.request(`/api/containers?tag_ids=${tagId}&tag_ids=red`) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain('tag_ids') + }) + }) + + describe('GET /api/containers location_id filter', () => { + it('lists only Containers stored at the Location or a Location under it', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer', description: 'Test' }) + const freezer = await createTestLocation(ctx.db, { + name: 'Freezer A', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const shelf = await createTestLocation(ctx.db, { name: 'Shelf 1', parentId: freezer.id, canContainCollections: true }) + const elsewhere = await createTestLocation(ctx.db, { + name: 'Freezer B', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const specimenId = (await createTestSpecimen(ctx.db, specimenType.id)).id + const tubeIn = async (locationId: number, name: string) => { + const cryoBox = await createTestCryovialBox(ctx.db, { name, locationId }) + const container = await createTestStorageContainer(ctx.db, { specimenId, unitId: unit.id }) + await createTestCryovialTube(ctx.db, { containerId: container.id, collectionId: cryoBox.id, barcode: `CV-${name}`, position: 'A1' }) + return container.id + } + const inFreezer = await tubeIn(freezer.id, 'F') + const onShelf = await tubeIn(shelf.id, 'S') + await tubeIn(elsewhere.id, 'E') + await createTestStorageContainer(ctx.db, { specimenId, unitId: unit.id }) + + const res = await ctx.request(`/api/containers?location_id=${freezer.id}&limit=50`) + expect(res.status).toBe(200) + const data = (await res.json()) as { containers: Array<{ id: number }>; pagination: { total: number } } + expect(data.containers.map((c) => c.id).sort((a, b) => a - b)).toEqual([inFreezer, onShelf]) + expect(data.pagination.total).toBe(2) + + const shelfOnly = (await (await ctx.request(`/api/containers?location_id=${shelf.id}`)).json()) as { + containers: Array<{ id: number }> + } + expect(shelfOnly.containers.map((c) => c.id)).toEqual([onShelf]) + }) + }) + describe('GET /api/containers/:id', () => { it('returns 404 for non-existent ID', async () => { const res = await ctx.request('/api/containers/99999') @@ -496,6 +557,18 @@ describe('Containers API', () => { expect(body.container.tags?.map((t) => t.id)).toEqual([otherTagId]) }) + it('returns 400 for an unknown unit on a Container whose type is unknown and leaves it unchanged', async () => { + const id = await seedContainer() + + const res = await ctx.request(`/api/containers/${id}`, { method: 'PATCH', json: { unitId: 999999, comment: 'changed' } }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe('Unknown unit id: 999999') + await expectUnchanged(id) + const row = await ctx.db.select().from(storageContainer).where(eq(storageContainer.id, id)).get() + expect(row?.unitId).toBe(unitId) + }) + it('returns 400 for an unknown Tag and leaves the Container unchanged', async () => { const id = await seedMicronix('MX-UNK', 'A01') diff --git a/packages/api/src/routes/__tests__/derivations.test.ts b/packages/api/src/routes/__tests__/derivations.test.ts index f2377e4f..b3cc5c27 100644 --- a/packages/api/src/routes/__tests__/derivations.test.ts +++ b/packages/api/src/routes/__tests__/derivations.test.ts @@ -264,6 +264,16 @@ describe('Derivations API', () => { expect((await loadDerivation(d.id))?.protocol).toBe('P2') }) + it('returns 400 for an empty PATCH body and leaves the Derivation unchanged (#120)', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'PATCH', json: {} }) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe( + 'Nothing to update: send derivationDate, protocol, notes or properties.', + ) + expect((await loadDerivation(d.id))?.protocol).toBe('P1') + }) + it('returns 403 to a member deleting a Derivation and keeps it', async () => { const d = await createDerivationRow() const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'DELETE', cookie: ctx.cookies.member }) diff --git a/packages/api/src/routes/__tests__/error-logs.test.ts b/packages/api/src/routes/__tests__/error-logs.test.ts index efe846c9..d361de0f 100644 --- a/packages/api/src/routes/__tests__/error-logs.test.ts +++ b/packages/api/src/routes/__tests__/error-logs.test.ts @@ -45,6 +45,16 @@ describe('Error Logs API', () => { ctx.cleanup() }) + describe('GET /api/error-logs oversized page (#120)', () => { + it('returns 400 naming page for a page above the highest page', async () => { + const res = await ctx.request('/api/error-logs?page=1000001&limit=10') + expect(res.status).toBe(400) + const data = (await res.json()) as { errorCode: string; details: Array<{ path: string[]; message: string }> } + expect(data.errorCode).toBe('VALIDATION_ERROR') + expect(data.details).toContainEqual(expect.objectContaining({ path: ['page'], message: 'The highest page is 1000000.' })) + }) + }) + describe('GET /api/error-logs', () => { it('returns 200 with admin', async () => { const res = await ctx.request('/api/error-logs') diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index 2508cf10..e253e066 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -125,6 +125,38 @@ describe('Export API', () => { expect(rowsAfter.length).toBe(rowsBefore.length) }) + it.each(['not json', '{"a":1}', '"barcode"'])('returns 400 for columns=%s', async (columns) => { + await seedTypedContainers('GCOL', { cryovial: false }) + + const res = await ctx.request(`/api/export/containers?study=GCOL&format=csv&columns=${encodeURIComponent(columns)}`) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain('Invalid columns') + }) + + it('exports the columns a JSON array names', async () => { + await seedTypedContainers('GCOK', { cryovial: false }) + + const res = await ctx.request( + `/api/export/containers?study=GCOK&format=csv&csv_bom=false&csv_line_ending=LF&columns=${encodeURIComponent('["barcode"]')}`, + ) + + expect(res.status).toBe(200) + }) + + it.each([ + ['subject_ids', 'abc'], + ['specimen_type_ids', '1x'], + ['tag_ids', 'red'], + ])('returns 400 for a non-numeric %s=%s', async (param, value) => { + await seedTypedContainers('GIDS', { cryovial: false }) + + const res = await ctx.request(`/api/export/containers?study=GIDS&count_only=true&${param}=${value}`) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + it.each([ ['date_from', 'garbage'], ['date_to', '5/7/2024'], diff --git a/packages/api/src/routes/__tests__/locations.test.ts b/packages/api/src/routes/__tests__/locations.test.ts index b5970793..7e91f98c 100644 --- a/packages/api/src/routes/__tests__/locations.test.ts +++ b/packages/api/src/routes/__tests__/locations.test.ts @@ -52,6 +52,30 @@ describe('Locations API', () => { }) }) + describe('GET /api/locations oversized page (#120)', () => { + it('returns 400 for an oversized page on the list', async () => { + const res = await ctx.request('/api/locations?page=99999999999999999999') + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain('page') + }) + + it.each(['plates_page', 'cryovial_boxes_page', 'boxes_page', 'bags_page'])( + 'returns 400 for an oversized %s on a Location', + async (param) => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { + name: 'Root', + storageTypeId: String(storageType.id), + canContainCollections: true, + path: 'Root', + }) + const res = await ctx.request(`/api/locations/${loc.id}?${param}=99999999999999999999`) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }, + ) + }) + describe('GET /api/locations/:id', () => { it('returns 200 and location when found', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) diff --git a/packages/api/src/routes/__tests__/specimens.test.ts b/packages/api/src/routes/__tests__/specimens.test.ts index 1eb864d1..f06c803f 100644 --- a/packages/api/src/routes/__tests__/specimens.test.ts +++ b/packages/api/src/routes/__tests__/specimens.test.ts @@ -95,6 +95,40 @@ describe('Specimens API', () => { }) }) + describe('GET /api/specimens oversized page (#120)', () => { + it.each([ + ['/api/specimens?page=99999999999999999999&limit=10', 'page'], + ])('returns 400 for %s', async (url, param) => { + const res = await ctx.request(url) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + }) + + describe('GET /api/specimens non-numeric id filters (#120)', () => { + it.each([ + ['subject_id', 'abc'], + ['control_batch_id', '1.5'], + ['specimen_type_id', 'x1'], + ])('returns 400 for %s=%s', async (param, value) => { + const res = await ctx.request(`/api/specimens?${param}=${value}`) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + + it('filters by a numeric subject_id', async () => { + const study = await createTestStudy(ctx.db, { title: 'S', shortCode: 'S1' }) + const subject = await createTestStudySubject(ctx.db, { studyId: study.id, name: 'Subj' }) + const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) + const mine = await createTestSpecimen(ctx.db, type.id, { studySubjectId: subject.id }) + await createTestSpecimen(ctx.db, type.id) + const res = await ctx.request(`/api/specimens?subject_id=${subject.id}`) + expect(res.status).toBe(200) + const data = (await res.json()) as { specimens: Array<{ id: number }> } + expect(data.specimens.map((s) => s.id)).toEqual([mine.id]) + }) + }) + describe('POST /api/specimens/bulk', () => { let testStudy: Awaited> let testSubject: Awaited> diff --git a/packages/api/src/routes/__tests__/statistics.test.ts b/packages/api/src/routes/__tests__/statistics.test.ts index a41ddc60..7b5a8dce 100644 --- a/packages/api/src/routes/__tests__/statistics.test.ts +++ b/packages/api/src/routes/__tests__/statistics.test.ts @@ -48,7 +48,7 @@ describe('Statistics API', () => { expect(data).toBeDefined() }) - it('filters by a numeric specimen_type_id and ignores a non-numeric one or location_id', async () => { + it('filters by a numeric specimen_type_id', async () => { await createTestStorageContainer(ctx.db) const otherType = await createTestSpecimenType(ctx.db, { name: 'Other type' }) const total = async (query: string) => { @@ -59,7 +59,16 @@ describe('Statistics API', () => { expect(await total('')).toBe(1) expect(await total(`?specimen_type_id=${otherType.id}`)).toBe(0) - expect(await total('?specimen_type_id=abc&location_id=abc')).toBe(1) + }) + + it.each([ + ['specimen_type_id', 'abc'], + ['location_id', 'abc'], + ['tag_ids', '1x'], + ])('returns 400 for a non-numeric %s=%s', async (param, value) => { + const res = await ctx.request(`/api/statistics?${param}=${value}`, { method: 'GET' }) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) }) it.each([ diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index 236385fe..3c599f55 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -74,6 +74,16 @@ describe('Studies API (list, get, create, update)', () => { }) }) + describe('GET /api/studies oversized page (#120)', () => { + it.each([ + ['/api/studies?page=99999999999999999999', 'page'], + ])('returns 400 for %s', async (url, param) => { + const res = await ctx.request(url) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + }) + describe('GET /api/studies/:id', () => { it('returns 200 and study when found', async () => { const studyRecord = await createTestStudy(ctx.db, { diff --git a/packages/api/src/routes/__tests__/subjects.test.ts b/packages/api/src/routes/__tests__/subjects.test.ts index 4c558a58..d8655534 100644 --- a/packages/api/src/routes/__tests__/subjects.test.ts +++ b/packages/api/src/routes/__tests__/subjects.test.ts @@ -73,6 +73,16 @@ describe('Subjects API', () => { }) }) + describe('GET /api/subjects oversized page (#120)', () => { + it.each([ + ['/api/subjects?page=99999999999999999999', 'page'], + ])('returns 400 for %s', async (url, param) => { + const res = await ctx.request(url) + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toContain(param) + }) + }) + describe('POST /api/subjects', () => { it('returns 201 and created subject when study exists', async () => { const studyRecord = await createTestStudy(ctx.db, { diff --git a/packages/api/src/routes/auth.ts b/packages/api/src/routes/auth.ts index 794d593e..cb119ff3 100644 --- a/packages/api/src/routes/auth.ts +++ b/packages/api/src/routes/auth.ts @@ -8,7 +8,7 @@ import type { Database } from '../db/client' import { getPasswordRequirements, getSessionSettings } from '../lib/settings' import { createAuthMiddleware, createAdminMiddleware } from '../middleware/auth' import { rateLimit } from '../middleware/rate-limit' -import { handleRouteError, NotFoundError, ValidationError } from '../lib/error-handler' +import { ConflictError, handleRouteError, NotFoundError, ValidationError } from '../lib/error-handler' import { toPublicUser, verifyLoginCredentials, @@ -17,6 +17,7 @@ import { deleteSessionByToken, deleteUserSessions, registerApprovedUser, + assertUserIdentityFree, BCRYPT_COST, } from '../lib/auth/auth-service' @@ -24,7 +25,10 @@ function publicUserResponse(user: ReturnType) { return { user: { ...user, username: user.username || undefined } } } import { utcNow } from '../lib/datetime' -import { requireParam } from '../lib/common-validators' +import { requireParam, setsNoField } from '../lib/common-validators' + +/** 400 for a PATCH or PUT whose body sets no field. */ +const NOTHING_TO_UPDATE = 'Nothing to update: send at least one field to change.' export function createAuthRoutes(database: Database, settingsDb?: Database) { const auth = new Hono() @@ -135,27 +139,7 @@ auth.post('/self-register', rateLimit(5, 60 * 1000), async (c) => { const schema = await createSelfRegisterSchema() const data = schema.parse(body) - const existingEmail = await database - .select() - .from(users) - .where(and(eq(users.email, data.email), isNull(users.deletedAt))) - .get() - - if (existingEmail) { - return c.json({ error: 'Email already in use' }, 400) - } - - if (data.username) { - const existingUsername = await database - .select() - .from(users) - .where(and(eq(users.username, data.username), isNull(users.deletedAt))) - .get() - - if (existingUsername) { - return c.json({ error: 'Username already in use' }, 400) - } - } + await assertUserIdentityFree(database, { email: data.email, username: data.username }) const passwordHash = await bcrypt.hash(data.password, BCRYPT_COST) const createdAt = utcNow() @@ -271,42 +255,11 @@ auth.patch('/me', authMiddleware, async (c) => { return c.json({ error: 'User not found' }, 404) } - // If email is being updated, check for duplicates (excluding current user) - if (data.email && data.email !== existing.email) { - const duplicate = await database - .select() - .from(users) - .where(and( - eq(users.email, data.email), - isNull(users.deletedAt), - ne(users.id, currentUser.id) - )) - .get() - - if (duplicate) { - return c.json({ error: 'Email already in use' }, 400) - } + if (setsNoField(data)) { + throw new ValidationError(NOTHING_TO_UPDATE) } - // If username is being updated, check for duplicates (excluding current user) - if (data.username !== undefined) { - // Allow setting username to null (clearing it) - if (data.username !== null && data.username !== existing.username) { - const duplicate = await database - .select() - .from(users) - .where(and( - eq(users.username, data.username), - isNull(users.deletedAt), - ne(users.id, currentUser.id) - )) - .get() - - if (duplicate) { - return c.json({ error: 'Username already in use' }, 400) - } - } - } + await assertUserIdentityFree(database, { email: data.email, username: data.username }, currentUser.id) const [updated] = await database .update(users) @@ -331,7 +284,7 @@ auth.patch('/me', authMiddleware, async (c) => { if (error instanceof z.ZodError) { return c.json({ error: 'Invalid input', details: error.issues }, 400) } - return c.json({ error: 'Internal server error' }, 500) + return handleRouteError(error, c) } }) @@ -536,42 +489,11 @@ auth.put('/users/:id', adminMiddleware, async (c) => { return c.json({ error: 'User not found' }, 404) } - // If email is being updated, check for duplicates (excluding soft-deleted users) - if (data.email && data.email !== existing.email) { - const duplicate = await database - .select() - .from(users) - .where(and( - eq(users.email, data.email), - isNull(users.deletedAt), - ne(users.id, id) - )) - .get() - - if (duplicate) { - return c.json({ error: 'Email already in use' }, 400) - } + if (setsNoField(data)) { + throw new ValidationError(NOTHING_TO_UPDATE) } - // If username is being updated, check for duplicates (excluding soft-deleted users) - if (data.username !== undefined) { - // Allow setting username to null (clearing it) - if (data.username !== null && data.username !== existing.username) { - const duplicate = await database - .select() - .from(users) - .where(and( - eq(users.username, data.username), - isNull(users.deletedAt), - ne(users.id, id) - )) - .get() - - if (duplicate) { - return c.json({ error: 'Username already in use' }, 400) - } - } - } + await assertUserIdentityFree(database, { email: data.email, username: data.username }, id) // Prevent removing the last admin if (data.role && data.role !== 'admin' && existing.role === 'admin') { @@ -615,7 +537,7 @@ auth.put('/users/:id', adminMiddleware, async (c) => { if (error instanceof z.ZodError) { return c.json({ error: 'Invalid input', details: error.issues }, 400) } - return c.json({ error: 'Internal server error' }, 500) + return handleRouteError(error, c) } }) @@ -799,21 +721,20 @@ auth.post('/users/:id/restore', adminMiddleware, async (c) => { return c.json({ error: 'User is not deleted' }, 400) } - // Check if email is still available (not taken by another active user) - const emailConflict = await database - .select() - .from(users) - .where(and( - eq(users.email, existing.email), - isNull(users.deletedAt), - ne(users.id, id) - )) - .get() - - if (emailConflict) { - return c.json({ - error: 'Cannot restore user: email is already in use by another user' - }, 400) + // Emails and usernames are unique across all users; check both before restoring. + for (const [label, column, value] of [ + ['email', users.email, existing.email], + ['username', users.username, existing.username], + ] as const) { + if (!value) continue + const holder = await database + .select({ id: users.id }) + .from(users) + .where(and(eq(column, value), ne(users.id, id))) + .get() + if (holder) { + throw new ConflictError(`Cannot restore user: ${label} is already in use by another user`) + } } // Restore: clear deletedAt @@ -841,7 +762,7 @@ auth.post('/users/:id/restore', adminMiddleware, async (c) => { }, }) } catch (error) { - return c.json({ error: 'Internal server error' }, 500) + return handleRouteError(error, c) } }) diff --git a/packages/api/src/routes/containers.ts b/packages/api/src/routes/containers.ts index 8d7349ab..5e556bde 100644 --- a/packages/api/src/routes/containers.ts +++ b/packages/api/src/routes/containers.ts @@ -1,6 +1,6 @@ import { Hono } from 'hono' import type { Database } from '../db/client' -import { storageContainer, micronixTube, cryovialTube, paper, staticWell, tag, storageContainerTag } from '../db/schema' +import { storageContainer, micronixTube, cryovialTube, paper, staticWell, tag, storageContainerTag, unit } from '../db/schema' import { eq, and, sql, inArray } from 'drizzle-orm' import { z } from 'zod' import { containerQuantityBoundsError } from '@sampledb/contract' @@ -21,7 +21,8 @@ import { } from '../lib/error-handler' import { withWriteTransaction } from '../db/write-transaction' import { validateUnitForContainerType } from '../lib/validation' -import { containerHasAllTags } from '../lib/container-selection' +import { containerAtLocation, containerHasAllTags } from '../lib/container-selection' +import { idListQueryParam, idQueryParam } from '../lib/query-params' /** * Create containers routes with database injection @@ -35,9 +36,9 @@ export function createContainersRoutes(database: Database): Hono { // List containers with filters containers.get('/', authMiddleware, async (c) => { try { - const specimenId = c.req.query('specimen_id') - const locationId = c.req.query('location_id') - const tagIds = c.req.queries('tag_ids')?.map(id => parseInt(id)).filter(id => !isNaN(id)) + const specimenId = idQueryParam('specimen_id', c.req.query('specimen_id')) + const locationId = idQueryParam('location_id', c.req.query('location_id')) + const tagIds = idListQueryParam('tag_ids', c.req.queries('tag_ids')) const page = validatePage(c.req.query('page')) const limit = await validateLimit(database, c.req.query('limit')) const offset = (page - 1) * limit @@ -47,16 +48,17 @@ containers.get('/', authMiddleware, async (c) => { const conditions = [] - if (specimenId) { - const id = parseInt(specimenId) - if (!isNaN(id)) { - conditions.push(eq(storageContainer.specimenId, id)) - } + if (specimenId !== undefined) { + conditions.push(eq(storageContainer.specimenId, specimenId)) } const tags = containerHasAllTags(tagIds ?? []) if (tags) conditions.push(tags) + if (locationId !== undefined) { + conditions.push(containerAtLocation(locationId)) + } + if (conditions.length > 0) { const whereClause = and(...conditions) as any query = query.where(whereClause) as any @@ -83,6 +85,7 @@ containers.get('/', authMiddleware, async (c) => { }, }) } catch (error: unknown) { + if (error instanceof ValidationError) return handleRouteError(error, c) return handleRouteError( new RouteError(500, containersFetchFailedBody('Failed to fetch containers', error)), c, @@ -260,6 +263,10 @@ containers.patch('/:id', memberMiddleware, async (c) => { const quantityError = containerQuantityBoundsError({ totalQuantity: container.totalQuantity, remainingQuantity }) if (quantityError) throw new ValidationError(quantityError) } + if (unitId !== undefined) { + const known = await tx.select({ id: unit.id }).from(unit).where(eq(unit.id, unitId)).get() + if (!known) throw new ValidationError(`Unknown unit id: ${unitId}`) + } if (unitId !== undefined && containerType) { const validation = await validateUnitForContainerType(tx, containerType, unitId) if (!validation.valid) { diff --git a/packages/api/src/routes/derivations.ts b/packages/api/src/routes/derivations.ts index f130dfba..372c8923 100644 --- a/packages/api/src/routes/derivations.ts +++ b/packages/api/src/routes/derivations.ts @@ -11,8 +11,8 @@ import { createDerivation } from '../lib/derivations' import { resolveSpecimenSource } from '../lib/specimens/provenance' import { createDerivationRequestSchema } from '../lib/derivation-schemas' import { createAdminMiddleware, createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' -import { requireParam } from '../lib/common-validators' -import { handleRouteError } from '../lib/error-handler' +import { requireParam, setsNoField } from '../lib/common-validators' +import { handleRouteError, ValidationError } from '../lib/error-handler' import { enrichContainersForApi } from '../lib/container-api-enrichment' import { mapEnrichedContainerToWire } from '../lib/container-wire-mapper' @@ -253,6 +253,9 @@ derivations.patch('/derivations/:id', adminMiddleware, async (c) => { properties: z.record(z.string(), z.any()).optional(), }) const data = schema.parse(body) + if (setsNoField(data)) { + throw new ValidationError('Nothing to update: send derivationDate, protocol, notes or properties.') + } const [updated] = await database .update(containerDerivation) diff --git a/packages/api/src/routes/error-logs.ts b/packages/api/src/routes/error-logs.ts index f4eeb849..b6330ee5 100644 --- a/packages/api/src/routes/error-logs.ts +++ b/packages/api/src/routes/error-logs.ts @@ -12,6 +12,7 @@ import { rateLimit } from '../middleware/rate-limit' import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' import { likeContains } from '../lib/sql-like' +import { MAX_PAGE } from '../lib/query-params' /** Header values copied into an error log row are cut to these lengths. */ const MAX_LOGGED_URL_LENGTH = 2_000 @@ -38,7 +39,7 @@ const errorLogQuerySchema = z.object({ return isNaN(num) ? undefined : num } return val - }, z.number().int().positive().optional()), + }, z.number().int().positive().max(MAX_PAGE, `The highest page is ${MAX_PAGE}.`).optional()), limit: z.preprocess((val) => { if (val === undefined || val === null || val === '') return undefined if (typeof val === 'string') { diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index 2bf6fa35..f3c22c30 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -26,12 +26,7 @@ import { exportInventoryCsv } from '../lib/export/inventory-csv' import { exportSpecimensCsv } from '../lib/export/specimens-csv' import type { ExportFilters, MultiStudyExportEntry } from '../lib/export/types' import { createAuthMiddleware } from '../middleware/auth' - -function parseTagIds(values: string[] | undefined): number[] | undefined { - if (!values || values.length === 0) return undefined - const ids = values.map((id) => parseInt(id, 10)).filter((id) => !Number.isNaN(id)) - return ids.length > 0 ? ids : undefined -} +import { idListQueryParam } from '../lib/query-params' function parseTagIdsFromBody(body: unknown): number[] | undefined { if (!body || typeof body !== 'object' || !('tag_ids' in body)) return undefined @@ -72,11 +67,21 @@ function handleExportFailure(c: Context, message: string, error: unknown): Respo return handleRouteError(new RouteError(500, { error: message, details }), c) } +/** Export columns: an array, or a JSON array in a string. Malformed JSON or a non-array throws a `ValidationError`. */ function parseColumnsParam(raw: unknown): string[] | undefined { if (!raw) return undefined if (Array.isArray(raw)) return raw.map(String) - if (typeof raw === 'string') return JSON.parse(raw) as string[] - return undefined + if (typeof raw !== 'string') return undefined + let parsed: unknown + try { + parsed = JSON.parse(raw) + } catch { + throw new ValidationError(`Invalid columns '${raw}': use a JSON array of column names, such as ["barcode"].`) + } + if (!Array.isArray(parsed)) { + throw new ValidationError(`Invalid columns '${raw}': use a JSON array of column names, such as ["barcode"].`) + } + return parsed.map(String) } function parseExportFormat(raw: unknown, fallback: ContainerExportFormat = 'json'): ContainerExportFormat { @@ -86,22 +91,18 @@ function parseExportFormat(raw: unknown, fallback: ContainerExportFormat = 'json function buildGetFilters(c: { req: { query: (key: string) => string | undefined; queries: (key: string) => string[] | undefined } }, studyCode: string): ExportFilters { const filters: ExportFilters = { study: studyCode } - const specimenTypeIds = c.req.queries('specimen_type_ids') - if (specimenTypeIds?.length) { - filters.specimen_type_ids = specimenTypeIds.map((id) => parseInt(id, 10)).filter((id) => !Number.isNaN(id)) - } + const specimenTypeIds = idListQueryParam('specimen_type_ids', c.req.queries('specimen_type_ids')) + if (specimenTypeIds) filters.specimen_type_ids = specimenTypeIds const containerTypes = c.req.queries('container_types') if (containerTypes?.length) { filters.container_types = containerTypes } - const subjectIds = c.req.queries('subject_ids') - if (subjectIds?.length) { - filters.subject_ids = subjectIds.map((id) => parseInt(id, 10)).filter((id) => !Number.isNaN(id)) - } + const subjectIds = idListQueryParam('subject_ids', c.req.queries('subject_ids')) + if (subjectIds) filters.subject_ids = subjectIds - const tagIds = parseTagIds(c.req.queries('tag_ids')) + const tagIds = idListQueryParam('tag_ids', c.req.queries('tag_ids')) if (tagIds) filters.tag_ids = tagIds if (c.req.query('date_from')) filters.date_from = c.req.query('date_from') as string @@ -183,9 +184,7 @@ export function createExportRoutes(database: Database): Hono { const result = await runSingleStudyContainerExportGet(database, { filters: buildGetFilters(c, studyCode), format: formatParam, - columns: c.req.query('columns') - ? parseColumnsParam(JSON.parse(c.req.query('columns') as string)) - : undefined, + columns: parseColumnsParam(c.req.query('columns')), csvOptions: parseCSVExportOptions(c.req.query()), userId: user?.id, countOnly, diff --git a/packages/api/src/routes/locations.ts b/packages/api/src/routes/locations.ts index f4de95f0..d5dd3cdd 100644 --- a/packages/api/src/routes/locations.ts +++ b/packages/api/src/routes/locations.ts @@ -159,19 +159,19 @@ locations.get('/:id', authMiddleware, async (c) => { const defaultLimit = 25 // Pagination parameters for each collection type - const platesPage = validatePage(c.req.query('plates_page') || '1') + const platesPage = validatePage(c.req.query('plates_page'), 'plates_page') const platesLimit = await validateLimit(database, c.req.query('plates_limit') || String(defaultLimit)) const platesOffset = (platesPage - 1) * platesLimit - const cryovialBoxesPage = validatePage(c.req.query('cryovial_boxes_page') || '1') + const cryovialBoxesPage = validatePage(c.req.query('cryovial_boxes_page'), 'cryovial_boxes_page') const cryovialBoxesLimit = await validateLimit(database, c.req.query('cryovial_boxes_limit') || String(defaultLimit)) const cryovialBoxesOffset = (cryovialBoxesPage - 1) * cryovialBoxesLimit - const boxesPage = validatePage(c.req.query('boxes_page') || '1') + const boxesPage = validatePage(c.req.query('boxes_page'), 'boxes_page') const boxesLimit = await validateLimit(database, c.req.query('boxes_limit') || String(defaultLimit)) const boxesOffset = (boxesPage - 1) * boxesLimit - const bagsPage = validatePage(c.req.query('bags_page') || '1') + const bagsPage = validatePage(c.req.query('bags_page'), 'bags_page') const bagsLimit = await validateLimit(database, c.req.query('bags_limit') || String(defaultLimit)) const bagsOffset = (bagsPage - 1) * bagsLimit From b1afdc83bcd056d869425fbeb1f8440dd9221fc7 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 19:48:47 -0700 Subject: [PATCH 30/54] fix(web): refresh other pages after Container, Subject, Control and Reference data writes Each write named in #119 changes records that Collection grids, Specimen lists, study and Subject pages, Controls and the dashboard show, so it now invalidates every query through invalidateAfterBulkWrite, passing the keys of deleted records so they are not refetched into a 404: - Container edit (useUpdateContainer), which refreshed only container queries - Derivation (new useCreateDerivation), which refreshed only the parent Container - adding a Container to a Specimen (new useAddSpecimenContainer), which refreshed only the Specimen's Containers - Subject merge (new useMergeSubjects), which refreshed only the study's Subjects - deleting a Specimen from a Control batch (new useDeleteControlBatchSpecimen), which refreshed only Controls queries - Reference data writes (invalidateAfterReferenceDataWrite), which refreshed only the Reference Data tab Study cards now count as unloaded once invalidated, and the Studies page refetches an invalidated card on screen, so they no longer keep old counts. hooks/README.md lists each write and what it invalidates. Closes #119 Co-Authored-By: Claude Opus 5.5 --- .../__tests__/helpers/mock-api-templates.ts | 1 + .../__tests__/helpers/query-invalidation.ts | 32 +++++ .../AddContainerForSpecimenModal.tsx | 10 +- .../components/ContainerDerivationModal.tsx | 14 +- .../web/src/components/SubjectMergeModal.tsx | 4 +- packages/web/src/hooks/README.md | 21 +++ .../hooks/__tests__/useContainers.test.tsx | 133 ++++++++++++++++-- .../src/hooks/__tests__/useControls.test.tsx | 87 ++++++++++++ .../hooks/__tests__/useReferenceData.test.tsx | 62 +++++++- .../src/hooks/__tests__/useStudies.test.tsx | 73 +++++++++- .../src/hooks/__tests__/useSubjects.test.tsx | 75 ++++++++++ packages/web/src/hooks/useContainers.ts | 39 ++++- packages/web/src/hooks/useControls.ts | 22 ++- packages/web/src/hooks/useReferenceData.ts | 77 ++++++---- packages/web/src/hooks/useStudies.ts | 55 ++++++-- packages/web/src/hooks/useSubjects.ts | 16 +++ packages/web/src/lib/query-client.ts | 8 +- packages/web/src/pages/ContainerDetail.tsx | 14 -- packages/web/src/pages/ControlBatchDetail.tsx | 6 +- packages/web/src/pages/ReferenceData.tsx | 22 +-- packages/web/src/pages/SpecimenDetail.tsx | 6 - packages/web/src/pages/StudyDetail.tsx | 2 +- .../__tests__/ControlBatchDetail.test.tsx | 36 +++++ .../pages/__tests__/ReferenceData.test.tsx | 83 +++++++++-- 24 files changed, 772 insertions(+), 126 deletions(-) create mode 100644 packages/web/src/__tests__/helpers/query-invalidation.ts create mode 100644 packages/web/src/hooks/__tests__/useControls.test.tsx diff --git a/packages/web/src/__tests__/helpers/mock-api-templates.ts b/packages/web/src/__tests__/helpers/mock-api-templates.ts index 4fd237dc..cbfee00e 100644 --- a/packages/web/src/__tests__/helpers/mock-api-templates.ts +++ b/packages/web/src/__tests__/helpers/mock-api-templates.ts @@ -75,6 +75,7 @@ export function studiesHooksMock(): MockApiOverrides { get: vi.fn(), getSubjects: vi.fn(), getSummary: vi.fn(), + getSummaries: vi.fn(), getTimeline: vi.fn(), create: vi.fn(), update: vi.fn(), diff --git a/packages/web/src/__tests__/helpers/query-invalidation.ts b/packages/web/src/__tests__/helpers/query-invalidation.ts new file mode 100644 index 00000000..f5103d45 --- /dev/null +++ b/packages/web/src/__tests__/helpers/query-invalidation.ts @@ -0,0 +1,32 @@ +import { QueryClient, QueryObserver, type QueryKey } from '@tanstack/react-query' +import { expect, vi } from 'vitest' + +/** A query client for hook tests: no retries, and cached data kept until the test ends. */ +export function createInvalidationTestClient(): QueryClient { + return new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) +} + +/** Cache data under each key, as pages elsewhere in the app would have, so a test can see which a write invalidates. */ +export function seedQueries(client: QueryClient, keys: readonly QueryKey[]): void { + for (const key of keys) client.setQueryData(key, { seeded: true }) +} + +/** The keys among `keys` whose cached query is not invalidated. */ +export function keysNotInvalidated(client: QueryClient, keys: readonly QueryKey[]): QueryKey[] { + return keys.filter((key) => client.getQueryState(key)?.isInvalidated !== true) +} + +/** + * Keep a query on screen, as a mounted page would, and resolve once its first fetch is done. + * The returned `queryFn` counts fetches, so a test can tell whether a write refetched the query. + */ +export async function observeQuery(client: QueryClient, key: QueryKey) { + const queryFn = vi.fn().mockResolvedValue({ observed: true }) + const observer = new QueryObserver(client, { queryKey: key, queryFn }) + const unsubscribe = observer.subscribe(() => {}) + await vi.waitFor(() => expect(queryFn).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => expect(client.getQueryState(key)?.status).toBe('success')) + return { queryFn, unsubscribe } +} diff --git a/packages/web/src/components/AddContainerForSpecimenModal.tsx b/packages/web/src/components/AddContainerForSpecimenModal.tsx index fda96c57..7d1bf48e 100644 --- a/packages/web/src/components/AddContainerForSpecimenModal.tsx +++ b/packages/web/src/components/AddContainerForSpecimenModal.tsx @@ -1,7 +1,7 @@ import { useState, useEffect } from 'react' import { Modal } from '../ui' import ContainerRegistration, { type ContainerData } from './ContainerRegistration' -import { specimensApi } from '../lib/api/specimens' +import { useAddSpecimenContainer } from '../hooks/useContainers' import { flatContainerRegistrationToWriteInput } from '../lib/specimen-container-payload' import { specimenTypesApi } from '../lib/api/reference-data'; import '../styles/subject-specimen.css' @@ -11,7 +11,8 @@ interface AddContainerForSpecimenModalProps { onClose: () => void specimenId: number specimenTypeId: number - onSuccess: () => void + /** Called after the Container is saved; `useAddSpecimenContainer` has already refreshed the queries. */ + onSuccess?: () => void } export default function AddContainerForSpecimenModal({ @@ -27,6 +28,7 @@ export default function AddContainerForSpecimenModal({ const [error, setError] = useState(null) const [allowedContainerTypes, setAllowedContainerTypes] = useState([]) const [allowedTypesLoading, setAllowedTypesLoading] = useState(false) + const addContainer = useAddSpecimenContainer() useEffect(() => { if (!isOpen || !specimenTypeId) return @@ -56,8 +58,8 @@ export default function AddContainerForSpecimenModal({ setError(null) setLoading(true) try { - await specimensApi.addContainer(specimenId, flatContainerRegistrationToWriteInput(containerData)) - onSuccess() + await addContainer.mutateAsync({ specimenId, container: flatContainerRegistrationToWriteInput(containerData) }) + onSuccess?.() onClose() } catch (err: unknown) { const message = diff --git a/packages/web/src/components/ContainerDerivationModal.tsx b/packages/web/src/components/ContainerDerivationModal.tsx index 1f1b92c0..a001d59f 100644 --- a/packages/web/src/components/ContainerDerivationModal.tsx +++ b/packages/web/src/components/ContainerDerivationModal.tsx @@ -1,6 +1,6 @@ import { useState, useEffect } from 'react' -import { derivationsApi } from '../lib/api/derivations'; import type { CreateDerivationPayload } from '../lib/api/derivations'; +import { useCreateDerivation } from '../hooks/useContainers' import { flatDerivationFormToWriteInput, type FlatDerivationContainerForm, @@ -28,7 +28,8 @@ interface ContainerDerivationModalProps { sublabel?: string specimenTypeName?: string } - onSuccess: () => void + /** Called after the Derivation is saved; `useCreateDerivation` has already refreshed the queries. */ + onSuccess?: () => void /** Pass a key that increments when opening to reset inner state; parent should increment in the open handler. */ openKey?: number } @@ -44,7 +45,7 @@ type ContainerDerivationModalContentProps = { parentContainerId: number parentContainer?: ContainerDerivationModalProps['parentContainer'] onClose: () => void - onSuccess: () => void + onSuccess?: () => void } function ContainerDerivationModalContent({ @@ -59,6 +60,7 @@ function ContainerDerivationModalContent({ const [specimenTypes, setSpecimenTypes] = useState([]) const [allowedContainerTypes, setAllowedContainerTypes] = useState([]) const [collectionOptions, setCollectionOptions] = useState([]) + const createDerivation = useCreateDerivation() const [formData, setFormData] = useState({ derivationType: 'dna_extraction', @@ -231,17 +233,17 @@ function ContainerDerivationModalContent({ properties: formData.properties, } - const response = await derivationsApi.createFromContainer(parentContainerId, payload) + const response = await createDerivation.mutateAsync({ parentContainerId, payload }) if (response.warnings.length > 0) { setWarnings(response.warnings) // Still show success but with warnings setTimeout(() => { - onSuccess() + onSuccess?.() onClose() }, 2000) } else { - onSuccess() + onSuccess?.() onClose() } } catch (err: unknown) { diff --git a/packages/web/src/components/SubjectMergeModal.tsx b/packages/web/src/components/SubjectMergeModal.tsx index 1455133d..b1a8d87b 100644 --- a/packages/web/src/components/SubjectMergeModal.tsx +++ b/packages/web/src/components/SubjectMergeModal.tsx @@ -3,6 +3,7 @@ import { subjectsApi } from '../lib/api/subjects'; import { studiesApi } from '../lib/api/studies'; import type { StudySubject } from '../lib/api/types'; import { useClickOutside } from '../hooks/useClickOutside' +import { useMergeSubjects } from '../hooks/useSubjects' import { Modal } from '../ui' import { parseDisplayDate } from '../lib/date-utils' @@ -38,6 +39,7 @@ function SubjectMergeModalContent({ const [sourceSelectedIndex, setSourceSelectedIndex] = useState(-1) const [targetSelectedIndex, setTargetSelectedIndex] = useState(-1) const [specimenCounts, setSpecimenCounts] = useState>(new Map()) + const mergeSubjects = useMergeSubjects() const sourceRef = useRef(null) const targetRef = useRef(null) const sourceResultsRef = useRef(null) @@ -131,7 +133,7 @@ function SubjectMergeModalContent({ try { setMerging(true) setError(null) - const response = await subjectsApi.merge(targetSubjectId, sourceSubjectId) + const response = await mergeSubjects.mutateAsync({ targetId: targetSubjectId, sourceId: sourceSubjectId }) // Show success message const { specimensTransferred, specimensMerged, containersMerged, totalContainersTransferred } = response diff --git a/packages/web/src/hooks/README.md b/packages/web/src/hooks/README.md index c3459256..509c2d62 100644 --- a/packages/web/src/hooks/README.md +++ b/packages/web/src/hooks/README.md @@ -17,6 +17,27 @@ After a **write**, ask: | `invalidateDashboardQueries` | Any create/update/delete that changes dashboard counts or widgets | | `invalidateSpecimenQueries` | Specimen create (via `useCreateSpecimen`) | | `invalidateQpcrExperimentQueries` | qPCR experiment create, delete, settings save, plate upload, results upload | +| `invalidateAfterBulkWrite` (`lib/query-client.ts`) | A write whose records many pages show; see the table below | +| `invalidateAfterReferenceDataWrite` | Any Reference data create, update or delete, and a specimen type's allowed container types | + +### Writes that invalidate every query + +These writes change records that Collection grids, Specimen lists, study and Subject pages, Controls and the dashboard all show, so a list of keys would miss some. They call `invalidateAfterBulkWrite`, which invalidates every query: queries on screen refetch now and the rest refetch when next used. Pass the keys of records the write deleted as `deleted`, so they are marked stale without refetching into a 404. + +| Write | Hook or call site | `deleted` | +|-------|-------------------|-----------| +| Container edit | `useUpdateContainer` | — | +| Derivation | `useCreateDerivation` | — | +| Add a Container to a Specimen | `useAddSpecimenContainer` | — | +| Subject merge | `useMergeSubjects` | source `subjectKeys.detail` | +| Delete a Specimen from a Control batch | `useDeleteControlBatchSpecimen` | `specimenKeys.detail`, `specimenKeys.containers` | +| Delete a Control batch | `ControlBatchDetail` | — (the page navigates away) | +| Reference data create, update, delete | `invalidateAfterReferenceDataWrite` (Reference Data page and the `useReferenceData` mutation hooks) | the item's detail key and, for a specimen type, its container types | +| Collection delete with contents | `useDeleteCollectionWithContents` | `collectionKeys.detail` | +| Study delete | `StudyDetail` | — (removes `studyKeys.detail` first) | +| Bulk import, derivations import, Control batch wizard, scan move, paper move, Collection move | the workflow hooks and pages | — | + +Study cards on the Studies page load through `useStudySummaryCards`, which refetches an invalidated card when it is on screen or next comes into view. ### Examples diff --git a/packages/web/src/hooks/__tests__/useContainers.test.tsx b/packages/web/src/hooks/__tests__/useContainers.test.tsx index c5329ed0..72e1e417 100644 --- a/packages/web/src/hooks/__tests__/useContainers.test.tsx +++ b/packages/web/src/hooks/__tests__/useContainers.test.tsx @@ -1,9 +1,25 @@ -import { describe, it, expect, vi } from 'vitest' +import { describe, it, expect, vi, beforeEach } from 'vitest' import { renderHook, waitFor } from '@testing-library/react' -import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { QueryClientProvider, type QueryClient } from '@tanstack/react-query' import type { ReactNode } from 'react' -import { containerKeys, useUpdateContainer } from '../useContainers' +import { + containerKeys, + useAddSpecimenContainer, + useCreateDerivation, + useUpdateContainer, +} from '../useContainers' +import { collectionKeys } from '../useCollections' +import { specimenKeys } from '../useSpecimens' +import { studyKeys } from '../useStudies' +import { dashboardKeys } from '../useDashboard' import { containersApi } from '../../lib/api/containers' +import { derivationsApi, type CreateDerivationPayload } from '../../lib/api/derivations' +import { specimensApi, type AddContainerData } from '../../lib/api/specimens' +import { + createInvalidationTestClient, + keysNotInvalidated, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' vi.mock('../../lib/api/containers', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -12,22 +28,55 @@ vi.mock('../../lib/api/containers', async () => { }) }) -function setup() { - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false, gcTime: 0 }, mutations: { retry: false } }, +vi.mock('../../lib/api/derivations', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('derivations', { + derivationsApi: { createFromContainer: vi.fn(), listFromContainer: vi.fn(), getSource: vi.fn() }, + }) +}) + +vi.mock('../../lib/api/specimens', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('specimens', { + specimensApi: { addContainer: vi.fn() }, }) +}) + +function setup() { + const queryClient = createInvalidationTestClient() const wrapper = ({ children }: { children: ReactNode }) => ( {children} ) return { queryClient, wrapper } } +/** Pages that show a Container: its detail, a Collection grid, its Specimen's Containers, and the dashboard. */ +const containerPages = [ + containerKeys.detail(5), + containerKeys.derivations(5), + collectionKeys.cryovialBox(1), + collectionKeys.micronixPlate(2), + specimenKeys.containers(7), + dashboardKeys.count('containers'), + dashboardKeys.secondary(), +] + +function seed(queryClient: QueryClient, extra: readonly (readonly unknown[])[] = []) { + const keys = [...containerPages, ...extra] + seedQueries(queryClient, keys) + return keys +} + +beforeEach(() => { + vi.clearAllMocks() +}) + describe('useUpdateContainer', () => { - it('saves the edit and invalidates every container query', async () => { + it('saves the edit and invalidates the Collection grids, Specimen Containers and dashboard that show it', async () => { const updated = { id: 5, containerType: 'paper' as const, sublabel: 'NEW' } vi.mocked(containersApi.update).mockResolvedValue(updated as never) const { queryClient, wrapper } = setup() - const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + const keys = seed(queryClient) const { result } = renderHook(() => useUpdateContainer(), { wrapper }) result.current.mutate({ id: 5, data: { sublabel: 'NEW' } }) @@ -35,18 +84,80 @@ describe('useUpdateContainer', () => { await waitFor(() => expect(result.current.isSuccess).toBe(true)) expect(containersApi.update).toHaveBeenCalledWith(5, { sublabel: 'NEW' }) expect(result.current.data).toEqual(updated) - expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: containerKeys.all }) + expect(keysNotInvalidated(queryClient, keys)).toEqual([]) }) it('invalidates nothing when the save fails', async () => { vi.mocked(containersApi.update).mockRejectedValue(new Error('Unknown tag id: 9')) const { queryClient, wrapper } = setup() - const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + const keys = seed(queryClient) const { result } = renderHook(() => useUpdateContainer(), { wrapper }) result.current.mutate({ id: 5, data: { tagIds: [9] } }) await waitFor(() => expect(result.current.isError).toBe(true)) - expect(invalidateSpy).not.toHaveBeenCalled() + expect(keysNotInvalidated(queryClient, keys)).toEqual(keys) + }) +}) + +describe('useCreateDerivation', () => { + const payload = { derivationType: 'dna_extraction', specimenTypeName: 'DNA' } as CreateDerivationPayload + + it('creates the Derivation and invalidates the Collection grids, Specimen lists, study summary and dashboard', async () => { + vi.mocked(derivationsApi.createFromContainer).mockResolvedValue({ warnings: [] } as never) + const { queryClient, wrapper } = setup() + const keys = seed(queryClient, [ + specimenKeys.list({}), + [...studyKeys.detail(3), 'summary'], + dashboardKeys.count('specimens'), + ]) + + const { result } = renderHook(() => useCreateDerivation(), { wrapper }) + result.current.mutate({ parentContainerId: 5, payload }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(derivationsApi.createFromContainer).toHaveBeenCalledWith(5, payload) + expect(keysNotInvalidated(queryClient, keys)).toEqual([]) + }) + + it('invalidates nothing when the Derivation fails', async () => { + vi.mocked(derivationsApi.createFromContainer).mockRejectedValue(new Error('Position taken')) + const { queryClient, wrapper } = setup() + const keys = seed(queryClient) + + const { result } = renderHook(() => useCreateDerivation(), { wrapper }) + result.current.mutate({ parentContainerId: 5, payload }) + + await waitFor(() => expect(result.current.isError).toBe(true)) + expect(keysNotInvalidated(queryClient, keys)).toEqual(keys) + }) +}) + +describe('useAddSpecimenContainer', () => { + const container = { containerType: 'cryovial_tube' } as AddContainerData + + it('adds the Container and invalidates the target Collection grid, the Specimen Containers and dashboard', async () => { + vi.mocked(specimensApi.addContainer).mockResolvedValue({ containerId: 11 }) + const { queryClient, wrapper } = setup() + const keys = seed(queryClient, [collectionKeys.box(4), specimenKeys.detail(7)]) + + const { result } = renderHook(() => useAddSpecimenContainer(), { wrapper }) + result.current.mutate({ specimenId: 7, container }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(specimensApi.addContainer).toHaveBeenCalledWith(7, container) + expect(keysNotInvalidated(queryClient, keys)).toEqual([]) + }) + + it('invalidates nothing when the add fails', async () => { + vi.mocked(specimensApi.addContainer).mockRejectedValue(new Error('Position taken')) + const { queryClient, wrapper } = setup() + const keys = seed(queryClient) + + const { result } = renderHook(() => useAddSpecimenContainer(), { wrapper }) + result.current.mutate({ specimenId: 7, container }) + + await waitFor(() => expect(result.current.isError).toBe(true)) + expect(keysNotInvalidated(queryClient, keys)).toEqual(keys) }) }) diff --git a/packages/web/src/hooks/__tests__/useControls.test.tsx b/packages/web/src/hooks/__tests__/useControls.test.tsx new file mode 100644 index 00000000..056c9ed3 --- /dev/null +++ b/packages/web/src/hooks/__tests__/useControls.test.tsx @@ -0,0 +1,87 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { renderHook, waitFor } from '@testing-library/react' +import { QueryClientProvider } from '@tanstack/react-query' +import type { ReactNode } from 'react' +import { controlKeys, useDeleteControlBatchSpecimen } from '../useControls' +import { collectionKeys } from '../useCollections' +import { specimenKeys } from '../useSpecimens' +import { dashboardKeys } from '../useDashboard' +import { controlsApi } from '../../lib/api/controls' +import { + createInvalidationTestClient, + keysNotInvalidated, + observeQuery, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' + +vi.mock('../../lib/api/controls', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('controls', { + controlsApi: { deleteSpecimenFromBatch: vi.fn() }, + }) +}) + +function setup() { + const queryClient = createInvalidationTestClient() + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ) + return { queryClient, wrapper } +} + +const batchPages = [ + collectionKeys.micronixPlate(4), + collectionKeys.sheet(6), + controlKeys.batchSummary(9), + controlKeys.overview(), + specimenKeys.list({}), + dashboardKeys.count('specimens'), +] + +describe('useDeleteControlBatchSpecimen', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('invalidates the Collection grids, the batch, the Controls overview and dashboard', async () => { + vi.mocked(controlsApi.deleteSpecimenFromBatch).mockResolvedValue({ message: 'Deleted' }) + const { queryClient, wrapper } = setup() + seedQueries(queryClient, batchPages) + + const { result } = renderHook(() => useDeleteControlBatchSpecimen(), { wrapper }) + result.current.mutate({ batchId: 9, specimenId: 30 }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(controlsApi.deleteSpecimenFromBatch).toHaveBeenCalledWith(9, 30) + expect(keysNotInvalidated(queryClient, batchPages)).toEqual([]) + }) + + it("marks the deleted Specimen's pages stale without refetching them", async () => { + vi.mocked(controlsApi.deleteSpecimenFromBatch).mockResolvedValue({ message: 'Deleted' }) + const { queryClient, wrapper } = setup() + const deletedDetail = await observeQuery(queryClient, specimenKeys.detail(30)) + const deletedContainers = await observeQuery(queryClient, specimenKeys.containers(30)) + const batch = await observeQuery(queryClient, controlKeys.batchSummary(9)) + + const { result } = renderHook(() => useDeleteControlBatchSpecimen(), { wrapper }) + result.current.mutate({ batchId: 9, specimenId: 30 }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(batch.queryFn).toHaveBeenCalledTimes(2) + expect(deletedDetail.queryFn).toHaveBeenCalledTimes(1) + expect(deletedContainers.queryFn).toHaveBeenCalledTimes(1) + for (const observed of [deletedDetail, deletedContainers, batch]) observed.unsubscribe() + }) + + it('invalidates nothing when the delete fails', async () => { + vi.mocked(controlsApi.deleteSpecimenFromBatch).mockRejectedValue(new Error('Not in this batch')) + const { queryClient, wrapper } = setup() + seedQueries(queryClient, batchPages) + + const { result } = renderHook(() => useDeleteControlBatchSpecimen(), { wrapper }) + result.current.mutate({ batchId: 9, specimenId: 30 }) + + await waitFor(() => expect(result.current.isError).toBe(true)) + expect(keysNotInvalidated(queryClient, batchPages)).toEqual(batchPages) + }) +}) diff --git a/packages/web/src/hooks/__tests__/useReferenceData.test.tsx b/packages/web/src/hooks/__tests__/useReferenceData.test.tsx index 7b4a282f..2bab5541 100644 --- a/packages/web/src/hooks/__tests__/useReferenceData.test.tsx +++ b/packages/web/src/hooks/__tests__/useReferenceData.test.tsx @@ -10,8 +10,19 @@ import { useUpdateSpecimenType, useDeleteSpecimenType, useStorageTypes, + useCreateTag, + useDeleteTag, + specimenTypeKeys, + tagKeys, } from '../useReferenceData' -import { specimenTypesApi, storageTypesApi } from '../../lib/api/reference-data' +import { exportWorkflowKeys } from '../useExportWorkflow' +import { specimenTypesApi, storageTypesApi, tagsApi } from '../../lib/api/reference-data' +import { + createInvalidationTestClient, + keysNotInvalidated, + observeQuery, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' import type { SpecimenType } from '../../lib/api/types' import type { StorageType } from '../../lib/api/reference-data' @@ -31,7 +42,12 @@ vi.mock('../../lib/api/reference-data', async () => { }, storageTypesApi: { list: vi.fn(), - } + }, + tagsApi: { + list: vi.fn(), + create: vi.fn(), + delete: vi.fn(), + }, }) }) @@ -286,5 +302,47 @@ describe('useReferenceData Hooks', () => { }) }) +describe('Tag writes refresh other pages', () => { + function setup() { + const queryClient = createInvalidationTestClient() + const wrapper = ({ children }: { children: ReactNode }) => ( + + {children} + + ) + return { queryClient, wrapper } + } + beforeEach(() => { + vi.clearAllMocks() + }) + + it('a new Tag invalidates the export Tag filter and the Specimen form catalogs', async () => { + vi.mocked(tagsApi.create).mockResolvedValue({ id: 4, name: 'Hemolyzed' } as never) + const { queryClient, wrapper } = setup() + const otherPages = [[...exportWorkflowKeys.referenceData(), 'tags'], tagKeys.list(), specimenTypeKeys.list()] + seedQueries(queryClient, otherPages) + + const { result } = renderHook(() => useCreateTag(), { wrapper }) + result.current.mutate({ name: 'Hemolyzed' } as never) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(keysNotInvalidated(queryClient, otherPages)).toEqual([]) + }) + it("a deleted Tag refreshes the Tag list without refetching the Tag's own query", async () => { + vi.mocked(tagsApi.delete).mockResolvedValue(undefined as never) + const { queryClient, wrapper } = setup() + const list = await observeQuery(queryClient, tagKeys.list()) + const deleted = await observeQuery(queryClient, tagKeys.detail(4)) + + const { result } = renderHook(() => useDeleteTag(), { wrapper }) + result.current.mutate(4) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + await waitFor(() => expect(list.queryFn).toHaveBeenCalledTimes(2)) + expect(deleted.queryFn).toHaveBeenCalledTimes(1) + list.unsubscribe() + deleted.unsubscribe() + }) +}) diff --git a/packages/web/src/hooks/__tests__/useStudies.test.tsx b/packages/web/src/hooks/__tests__/useStudies.test.tsx index 10772eab..94f4863d 100644 --- a/packages/web/src/hooks/__tests__/useStudies.test.tsx +++ b/packages/web/src/hooks/__tests__/useStudies.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, beforeEach, vi } from 'vitest' -import { renderHook, waitFor } from '@testing-library/react' +import { act, renderHook, waitFor } from '@testing-library/react' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' import { ReactNode } from 'react' import { ToastProvider } from '../../contexts/ToastContext' @@ -9,9 +9,11 @@ import { useStudySubjects, useCreateStudy, useUpdateStudy, + useStudySummaryCards, studyKeys, } from '../useStudies' -import { studiesApi } from '../../lib/api/studies' +import { invalidateAfterBulkWrite } from '../../lib/query-client' +import { studiesApi, type StudySummaryBasic } from '../../lib/api/studies' vi.mock('../../lib/api/studies', async () => { const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') @@ -275,5 +277,72 @@ describe('useStudies Hooks', () => { }) }) +describe('useStudySummaryCards', () => { + function summary(studyId: number, totalSpecimens: number) { + return { studyId, totalSpecimens } as unknown as StudySummaryBasic + } + function setup() { + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }) + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ) + return { queryClient, ...renderHook(() => useStudySummaryCards(), { wrapper }) } + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('loads a card once while its summary is current', async () => { + vi.mocked(studiesApi.getSummaries).mockResolvedValue({ summaries: [summary(1, 4)] }) + const { result } = setup() + + act(() => result.current.prefetch([1])) + await waitFor(() => expect(result.current.getCardState(1).summary).toEqual(summary(1, 4))) + act(() => result.current.prefetch([1])) + + expect(studiesApi.getSummaries).toHaveBeenCalledTimes(1) + }) + it('refetches a loaded card when a write invalidates it', async () => { + vi.mocked(studiesApi.getSummaries) + .mockResolvedValueOnce({ summaries: [summary(1, 4), summary(2, 0)] }) + .mockResolvedValueOnce({ summaries: [summary(1, 5), summary(2, 1)] }) + const { queryClient, result } = setup() + + act(() => result.current.prefetch([1, 2])) + await waitFor(() => expect(result.current.getCardState(1).summary).toEqual(summary(1, 4))) + + await act(() => invalidateAfterBulkWrite(queryClient)) + + await waitFor(() => expect(result.current.getCardState(1).summary).toEqual(summary(1, 5))) + expect(result.current.getCardState(2).summary).toEqual(summary(2, 1)) + expect(studiesApi.getSummaries).toHaveBeenCalledTimes(2) + expect(studiesApi.getSummaries).toHaveBeenLastCalledWith([1, 2]) + }) + + it('refetches an invalidated card that comes into view later', async () => { + vi.mocked(studiesApi.getSummaries) + .mockResolvedValueOnce({ summaries: [summary(1, 4)] }) + .mockResolvedValueOnce({ summaries: [summary(1, 5)] }) + const { queryClient, result, unmount } = setup() + act(() => result.current.prefetch([1])) + await waitFor(() => expect(result.current.getCardState(1).summary).toEqual(summary(1, 4))) + unmount() + + // The write happens on another page; the Studies page is not mounted. + await invalidateAfterBulkWrite(queryClient) + const remounted = renderHook(() => useStudySummaryCards(), { + wrapper: ({ children }: { children: ReactNode }) => ( + {children} + ), + }) + act(() => remounted.result.current.prefetch([1])) + + await waitFor(() => expect(remounted.result.current.getCardState(1).summary).toEqual(summary(1, 5))) + expect(studiesApi.getSummaries).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/web/src/hooks/__tests__/useSubjects.test.tsx b/packages/web/src/hooks/__tests__/useSubjects.test.tsx index c9d2d0eb..3f3707ce 100644 --- a/packages/web/src/hooks/__tests__/useSubjects.test.tsx +++ b/packages/web/src/hooks/__tests__/useSubjects.test.tsx @@ -8,8 +8,20 @@ import { useSubjectSummary, useCreateSubject, useUpdateSubject, + useMergeSubjects, + subjectKeys, } from '../useSubjects' +import { studyKeys } from '../useStudies' +import { specimenKeys } from '../useSpecimens' +import { collectionKeys } from '../useCollections' +import { dashboardKeys } from '../useDashboard' import { subjectsApi } from '../../lib/api/subjects' +import { + createInvalidationTestClient, + keysNotInvalidated, + observeQuery, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' // Mock the API module vi.mock('../../lib/api/subjects', async () => { @@ -191,5 +203,68 @@ describe('useSubjects Hooks', () => { }) }) +describe('useMergeSubjects', () => { + const studyPages = [ + [...studyKeys.detail(1), 'subjects', { page: 1 }], + [...studyKeys.detail(1), 'summary'], + [...studyKeys.detail(1), 'timeline'], + subjectKeys.summary(20), + specimenKeys.list({}), + collectionKeys.cryovialBox(4), + dashboardKeys.count('subjects'), + ] + + function setup() { + const queryClient = createInvalidationTestClient() + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ) + return { queryClient, wrapper } + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + it('invalidates the study Subjects, summary, timeline, target Subject and dashboard', async () => { + vi.mocked(subjectsApi.merge).mockResolvedValue({ success: true } as never) + const { queryClient, wrapper } = setup() + seedQueries(queryClient, studyPages) + + const { result } = renderHook(() => useMergeSubjects(), { wrapper }) + result.current.mutate({ targetId: 20, sourceId: 21 }) + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(subjectsApi.merge).toHaveBeenCalledWith(20, 21) + expect(keysNotInvalidated(queryClient, studyPages)).toEqual([]) + }) + + it("marks the deleted source Subject's pages stale without refetching them", async () => { + vi.mocked(subjectsApi.merge).mockResolvedValue({ success: true } as never) + const { queryClient, wrapper } = setup() + const source = await observeQuery(queryClient, subjectKeys.summary(21)) + const target = await observeQuery(queryClient, subjectKeys.summary(20)) + + const { result } = renderHook(() => useMergeSubjects(), { wrapper }) + result.current.mutate({ targetId: 20, sourceId: 21 }) + + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(target.queryFn).toHaveBeenCalledTimes(2) + expect(source.queryFn).toHaveBeenCalledTimes(1) + expect(queryClient.getQueryState(subjectKeys.summary(21))?.isInvalidated).toBe(true) + source.unsubscribe() + target.unsubscribe() + }) + + it('invalidates nothing when the merge fails', async () => { + vi.mocked(subjectsApi.merge).mockRejectedValue(new Error('Subjects are in different studies')) + const { queryClient, wrapper } = setup() + seedQueries(queryClient, studyPages) + const { result } = renderHook(() => useMergeSubjects(), { wrapper }) + result.current.mutate({ targetId: 20, sourceId: 21 }) + + await waitFor(() => expect(result.current.isError).toBe(true)) + expect(keysNotInvalidated(queryClient, studyPages)).toEqual(studyPages) + }) +}) diff --git a/packages/web/src/hooks/useContainers.ts b/packages/web/src/hooks/useContainers.ts index 669e3899..ca63625f 100644 --- a/packages/web/src/hooks/useContainers.ts +++ b/packages/web/src/hooks/useContainers.ts @@ -1,6 +1,8 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { containersApi, type ContainerDetail, type ContainerUpdate } from '../lib/api/containers' -import { derivationsApi } from '../lib/api/derivations' +import { derivationsApi, type CreateDerivationPayload } from '../lib/api/derivations' +import { specimensApi, type AddContainerData } from '../lib/api/specimens' +import { invalidateAfterBulkWrite } from '../lib/query-client' export const containerKeys = { all: ['containers'] as const, @@ -72,14 +74,43 @@ export function useContainerDerivationTree(containerId: number) { } /** - * Save a Container edit. Invalidates every container query: the edit changes the detail, and - * derivation trees and lists show the Container's quantity, Tags and identifier too. + * Save a Container edit. Collection grids, the Specimen's Containers, derivation trees, exports and + * dashboard statistics show a Container's quantity, Tags and identifier, so every query is invalidated. * Shows no toast; the caller reports the error. */ export function useUpdateContainer() { const queryClient = useQueryClient() return useMutation({ mutationFn: ({ id, data }: { id: number; data: ContainerUpdate }) => containersApi.update(id, data), - onSuccess: () => queryClient.invalidateQueries({ queryKey: containerKeys.all }), + onSuccess: () => invalidateAfterBulkWrite(queryClient), + }) +} + +/** + * Derive a Specimen and Container from a parent Container. The write adds a Specimen and a Container + * to a Collection and can reduce the parent's quantity, which Collection grids, Specimen lists, study + * and Subject summaries and the dashboard show, so every query is invalidated. + * Shows no toast; the caller reports the error. + */ +export function useCreateDerivation() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: ({ parentContainerId, payload }: { parentContainerId: number; payload: CreateDerivationPayload }) => + derivationsApi.createFromContainer(parentContainerId, payload), + onSuccess: () => invalidateAfterBulkWrite(queryClient), + }) +} + +/** + * Add a Container to an existing Specimen. The Container lands in a Collection's grid and counts on + * the Specimen, study, Subject and dashboard, so every query is invalidated. + * Shows no toast; the caller reports the error. + */ +export function useAddSpecimenContainer() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: ({ specimenId, container }: { specimenId: number; container: AddContainerData }) => + specimensApi.addContainer(specimenId, container), + onSuccess: () => invalidateAfterBulkWrite(queryClient), }) } diff --git a/packages/web/src/hooks/useControls.ts b/packages/web/src/hooks/useControls.ts index 8579618f..43e92bed 100644 --- a/packages/web/src/hooks/useControls.ts +++ b/packages/web/src/hooks/useControls.ts @@ -1,10 +1,12 @@ -import { useQuery } from '@tanstack/react-query' +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query' import { controlsApi, type ControlBatch, type ControlDefinition } from '../lib/api/controls' import { strainsApi } from '../lib/api/reference-data' import type { Strain } from '../lib/api/reference-data' import { getCompositionKey } from '../lib/composition-key' import { localToday } from '../lib/date-utils' import { getHttpStatus } from '../ui/async' +import { invalidateAfterBulkWrite } from '../lib/query-client' +import { specimenKeys } from './useSpecimens' export const controlKeys = { all: ['blood-controls'] as const, @@ -147,3 +149,21 @@ export function useCompositionDefinitionsByKey(compositionKey: string | null | u enabled: !!compositionKey, }) } + +/** + * Delete a Specimen and its Containers from a Control batch. The delete empties Collection grid + * positions and changes the batch, the Controls overview, Specimen lists and the dashboard, so every + * query is invalidated; the deleted Specimen's own queries are not refetched. + * Shows no toast; the caller reports the error. + */ +export function useDeleteControlBatchSpecimen() { + const queryClient = useQueryClient() + return useMutation({ + mutationFn: ({ batchId, specimenId }: { batchId: number; specimenId: number }) => + controlsApi.deleteSpecimenFromBatch(batchId, specimenId), + onSuccess: (_result, { specimenId }) => + invalidateAfterBulkWrite(queryClient, { + deleted: [specimenKeys.detail(specimenId), specimenKeys.containers(specimenId)], + }), + }) +} diff --git a/packages/web/src/hooks/useReferenceData.ts b/packages/web/src/hooks/useReferenceData.ts index e1dd3201..f5874c63 100644 --- a/packages/web/src/hooks/useReferenceData.ts +++ b/packages/web/src/hooks/useReferenceData.ts @@ -1,20 +1,47 @@ -import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' +import { useQuery, useMutation, useQueryClient, type QueryClient, type QueryKey } from '@tanstack/react-query' import { specimenTypesApi, tagsApi, storageTypesApi, strainsApi } from '../lib/api/reference-data' import type { SpecimenType } from '../lib/api/types' import type { Tag, StorageType, Strain } from '../lib/api/reference-data' import { useToast } from '../contexts/ToastContext' +import type { ReferenceDataType } from '../config/reference-data-config' +import { invalidateAfterBulkWrite } from '../lib/query-client' +import { referenceDataPageKeys } from './useReferenceDataPage' // Generic reference data keys factory -function createReferenceDataKeys(type: T) { +function createReferenceDataKeys(type: T) { return { all: [type] as const, lists: () => [type, 'list'] as const, list: () => [type, 'list'] as const, details: () => [type, 'detail'] as const, - detail: (id: number) => [type, 'detail', id] as const, + detail: (id: number) => referenceDataDetailKey(type, id), } } +function referenceDataDetailKey(type: T, id: number) { + return [type, 'detail', id] as const +} + +/** + * Refresh queries after a Reference data write. Reference data appears on most pages (a Tag on + * Containers and the export filters, a specimen type on Specimens, Collections and the Specimen + * form, a strain on Controls), so every query is invalidated. A deleted item's own queries are + * not refetched. + */ +export function invalidateAfterReferenceDataWrite( + queryClient: QueryClient, + deletedItem?: { type: ReferenceDataType; id: number } +) { + const deleted: QueryKey[] = [] + if (deletedItem) { + deleted.push(referenceDataDetailKey(deletedItem.type, deletedItem.id)) + if (deletedItem.type === 'specimen-types') { + deleted.push(referenceDataPageKeys.containerTypes(deletedItem.id)) + } + } + return invalidateAfterBulkWrite(queryClient, { deleted }) +} + export const specimenTypeKeys = createReferenceDataKeys('specimen-types') export const tagKeys = createReferenceDataKeys('tags') export const storageTypeKeys = createReferenceDataKeys('storage-types') @@ -95,7 +122,7 @@ export function useCreateSpecimenType() { } }, onSuccess: () => { - queryClient.invalidateQueries({ queryKey: specimenTypeKeys.lists() }) + void invalidateAfterReferenceDataWrite(queryClient) success('Specimen type created successfully') }, onError: () => { @@ -117,9 +144,8 @@ export function useUpdateSpecimenType() { throw err } }, - onSuccess: (data) => { - queryClient.invalidateQueries({ queryKey: specimenTypeKeys.detail(data.id) }) - queryClient.invalidateQueries({ queryKey: specimenTypeKeys.lists() }) + onSuccess: () => { + void invalidateAfterReferenceDataWrite(queryClient) success('Specimen type updated successfully') }, onError: () => { @@ -141,8 +167,8 @@ export function useDeleteSpecimenType() { throw err } }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: specimenTypeKeys.lists() }) + onSuccess: (_result, id) => { + void invalidateAfterReferenceDataWrite(queryClient, { type: 'specimen-types', id }) success('Specimen type deleted successfully') }, onError: () => { @@ -184,7 +210,7 @@ export function useCreateTag() { } }, onSuccess: () => { - queryClient.invalidateQueries({ queryKey: tagKeys.lists() }) + void invalidateAfterReferenceDataWrite(queryClient) success('Tag created successfully') }, onError: () => { @@ -206,9 +232,8 @@ export function useUpdateTag() { throw err } }, - onSuccess: (data) => { - queryClient.invalidateQueries({ queryKey: tagKeys.detail(data.id) }) - queryClient.invalidateQueries({ queryKey: tagKeys.lists() }) + onSuccess: () => { + void invalidateAfterReferenceDataWrite(queryClient) success('Tag updated successfully') }, onError: () => { @@ -230,8 +255,8 @@ export function useDeleteTag() { throw err } }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: tagKeys.lists() }) + onSuccess: (_result, id) => { + void invalidateAfterReferenceDataWrite(queryClient, { type: 'tags', id }) success('Tag deleted successfully') }, onError: () => { @@ -272,7 +297,7 @@ export function useCreateStorageType() { } }, onSuccess: () => { - queryClient.invalidateQueries({ queryKey: storageTypeKeys.lists() }) + void invalidateAfterReferenceDataWrite(queryClient) success('Storage type created successfully') }, onError: () => { @@ -294,9 +319,8 @@ export function useUpdateStorageType() { throw err } }, - onSuccess: (data) => { - queryClient.invalidateQueries({ queryKey: storageTypeKeys.detail(data.id) }) - queryClient.invalidateQueries({ queryKey: storageTypeKeys.lists() }) + onSuccess: () => { + void invalidateAfterReferenceDataWrite(queryClient) success('Storage type updated successfully') }, onError: () => { @@ -318,8 +342,8 @@ export function useDeleteStorageType() { throw err } }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: storageTypeKeys.lists() }) + onSuccess: (_result, id) => { + void invalidateAfterReferenceDataWrite(queryClient, { type: 'storage-types', id }) success('Storage type deleted successfully') }, onError: () => { @@ -366,7 +390,7 @@ export function useCreateStrain() { } }, onSuccess: () => { - queryClient.invalidateQueries({ queryKey: strainKeys.lists() }) + void invalidateAfterReferenceDataWrite(queryClient) success('Strain created successfully') }, onError: () => { @@ -388,9 +412,8 @@ export function useUpdateStrain() { throw err } }, - onSuccess: (data) => { - queryClient.invalidateQueries({ queryKey: strainKeys.detail(data.id) }) - queryClient.invalidateQueries({ queryKey: strainKeys.lists() }) + onSuccess: () => { + void invalidateAfterReferenceDataWrite(queryClient) success('Strain updated successfully') }, onError: () => { @@ -412,8 +435,8 @@ export function useDeleteStrain() { throw err } }, - onSuccess: () => { - queryClient.invalidateQueries({ queryKey: strainKeys.lists() }) + onSuccess: (_result, id) => { + void invalidateAfterReferenceDataWrite(queryClient, { type: 'strains', id }) success('Strain deleted successfully') }, onError: () => { diff --git a/packages/web/src/hooks/useStudies.ts b/packages/web/src/hooks/useStudies.ts index 2d9f8527..9648c143 100644 --- a/packages/web/src/hooks/useStudies.ts +++ b/packages/web/src/hooks/useStudies.ts @@ -26,15 +26,15 @@ export const studyKeys = { /** Page size for loading every study; the API clamps it to its configured maximum. */ const ALL_STUDIES_PAGE_LIMIT = 1000 -const SUMMARY_CARD_STALE_MS = 5 * 60 * 1000 function summaryBatchKey(ids: number[]) { return [...studyKeys.all, 'summaries-batch', [...ids].sort((a, b) => a - b)] as const } +/** A card is resolved once loaded, until a write invalidates it. */ function isSummaryCardResolved(queryClient: QueryClient, id: number): boolean { const state = queryClient.getQueryState(studyKeys.summaryCard(id)) - return state?.status === 'success' || state?.status === 'error' + return (state?.status === 'success' || state?.status === 'error') && !state.isInvalidated } export async function prefetchStudySummaryCards( @@ -57,7 +57,8 @@ export async function prefetchStudySummaryCards( .forEach((id) => queryClient.setQueryData(studyKeys.summaryCard(id), null)) return summaries }, - staleTime: SUMMARY_CARD_STALE_MS, + // Callers pass only unresolved cards, so fetch even when an earlier batch of the same ids is cached. + staleTime: 0, }) } @@ -66,21 +67,12 @@ export function useStudySummaryCards() { const queryClient = useQueryClient() const [cacheTick, setCacheTick] = useState(0) const loadingIdsRef = useRef(new Set()) - - useEffect(() => { - return queryClient.getQueryCache().subscribe((event) => { - const key = event.query.queryKey - if ( - key?.[0] === studyKeys.all[0] && - (key[1] === 'summary-card' || key[1] === 'summaries-batch') - ) { - setCacheTick((n) => n + 1) - } - }) - }, [queryClient]) + /** Cards this page has asked for, so an invalidated one on screen refetches. */ + const requestedIdsRef = useRef(new Set()) const prefetch = useCallback( (studyIds: number[]) => { + studyIds.forEach((id) => requestedIdsRef.current.add(id)) const toFetch = studyIds.filter((id) => !isSummaryCardResolved(queryClient, id)) if (toFetch.length === 0) return toFetch.forEach((id) => loadingIdsRef.current.add(id)) @@ -93,6 +85,39 @@ export function useStudySummaryCards() { [queryClient] ) + useEffect(() => { + let invalidatedIds: number[] = [] + const unsubscribe = queryClient.getQueryCache().subscribe((event) => { + const key = event.query.queryKey + if ( + key[0] !== studyKeys.all[0] || + (key[1] !== 'summary-card' && key[1] !== 'summaries-batch') + ) { + return + } + setCacheTick((n) => n + 1) + const id = key[2] + if ( + key[1] === 'summary-card' && + typeof id === 'number' && + requestedIdsRef.current.has(id) && + event.type === 'updated' && + event.action.type === 'invalidate' + ) { + // An invalidation marks every card in one pass; refetch them in one batch. + if (invalidatedIds.length === 0) { + queueMicrotask(() => { + const ids = invalidatedIds + invalidatedIds = [] + prefetch(ids) + }) + } + invalidatedIds.push(id) + } + }) + return unsubscribe + }, [queryClient, prefetch]) + const getCardState = useCallback( (studyId: number): { summary?: StudySummaryBasic | null; loading: boolean } => { void cacheTick diff --git a/packages/web/src/hooks/useSubjects.ts b/packages/web/src/hooks/useSubjects.ts index 18d6d311..8dbc77d0 100644 --- a/packages/web/src/hooks/useSubjects.ts +++ b/packages/web/src/hooks/useSubjects.ts @@ -2,6 +2,7 @@ import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query' import { subjectsApi, type SubjectSummaryResponse } from '../lib/api/subjects' import { studyKeys } from './useStudies' import { specimenKeys } from './useSpecimens' +import { invalidateAfterBulkWrite } from '../lib/query-client' export const subjectKeys = { all: ['subjects'] as const, @@ -79,3 +80,18 @@ export function invalidateSubjectDetail(queryClient: ReturnType + subjectsApi.merge(targetId, sourceId), + onSuccess: (_result, { sourceId }) => + invalidateAfterBulkWrite(queryClient, { deleted: [subjectKeys.detail(sourceId)] }), + }) +} diff --git a/packages/web/src/lib/query-client.ts b/packages/web/src/lib/query-client.ts index 773ade3f..efa48ccd 100644 --- a/packages/web/src/lib/query-client.ts +++ b/packages/web/src/lib/query-client.ts @@ -22,9 +22,11 @@ export const queryClient = new QueryClient({ }) /** - * Mark every cached query stale after a write that touches many kinds of records - * (bulk import, move, cascade delete). Queries on screen refetch now; the rest - * refetch when next used, so no page shows pre-write data for the 5-minute staleTime. + * Mark every cached query stale after a write whose records other pages show: a bulk + * import, move or cascade delete, and single writes such as a Container edit, a + * Derivation, a Subject merge or a Reference data edit (the table in + * `hooks/README.md` lists them). Queries on screen refetch now; the rest refetch when + * next used, so no page shows pre-write data for the 5-minute staleTime. * * `deleted` lists the keys of records the write removed (for example a deleted * Collection's detail key). Their queries are marked stale without refetching, so diff --git a/packages/web/src/pages/ContainerDetail.tsx b/packages/web/src/pages/ContainerDetail.tsx index 199205bb..a80f9465 100644 --- a/packages/web/src/pages/ContainerDetail.tsx +++ b/packages/web/src/pages/ContainerDetail.tsx @@ -1,12 +1,10 @@ import { useState } from 'react' import { useParams, Link, useNavigate } from 'react-router-dom' -import { useQueryClient } from '@tanstack/react-query' import type { Derivation } from '../lib/api/derivations' import EntityBreadcrumbs from '../components/EntityBreadcrumbs' import { getContainerTypeIcon, getContainerTypeName, getSpecimenTypeIcon } from '../lib/icons' import type { ContainerDetail as ContainerDetailData } from '../lib/api/containers' import { - containerKeys, useContainer, useContainerDerivationTree, useContainerSource, @@ -37,7 +35,6 @@ function statusColor(name: string): string { export default function ContainerDetail() { const { id } = useParams<{ id: string }>() const navigate = useNavigate() - const queryClient = useQueryClient() const { canWrite } = useUser() const containerId = id != null ? parseInt(id, 10) : NaN const containerQuery = useContainer(containerId) @@ -61,16 +58,6 @@ export default function ContainerDetail() { const [derivationModalKey, setDerivationModalKey] = useState(0) const [showEditModal, setShowEditModal] = useState(false) - const refreshContainer = () => { - void queryClient.invalidateQueries({ queryKey: containerKeys.detail(containerId) }) - void queryClient.invalidateQueries({ queryKey: containerKeys.derivations(containerId) }) - void queryClient.invalidateQueries({ queryKey: containerKeys.source(containerId) }) - } - - const handleDerivationCreated = () => { - refreshContainer() - } - if (detailStatus === 'loading') { return (
@@ -688,7 +675,6 @@ export default function ContainerDetail() { sublabel: paperSublabel, specimenTypeName: specimen?.specimenType?.name, }} - onSuccess={handleDerivationCreated} openKey={derivationModalKey} /> )} diff --git a/packages/web/src/pages/ControlBatchDetail.tsx b/packages/web/src/pages/ControlBatchDetail.tsx index 9b43458e..d054eb29 100644 --- a/packages/web/src/pages/ControlBatchDetail.tsx +++ b/packages/web/src/pages/ControlBatchDetail.tsx @@ -4,7 +4,7 @@ import { useQueryClient } from '@tanstack/react-query' import { useHotkey } from '../hooks/useHotkey' import { controlsApi } from '../lib/api/controls' import { invalidateAfterBulkWrite } from '../lib/query-client' -import { controlKeys, useControlBatchSummary } from '../hooks/useControls' +import { controlKeys, useControlBatchSummary, useDeleteControlBatchSpecimen } from '../hooks/useControls' import EntityBreadcrumbs from '../components/EntityBreadcrumbs' import SimpleTimeline from '../components/SimpleTimeline' import { getContainerTypeIcon, getContainerTypeName } from '../lib/icons' @@ -22,6 +22,7 @@ export default function ControlBatchDetail() { const { canWrite, isAdmin } = useUser() const batchId = id != null ? parseInt(id, 10) : NaN const summaryQuery = useControlBatchSummary(batchId) + const deleteSpecimen = useDeleteControlBatchSpecimen() const summaryStatus = fromQuery(summaryQuery) const summaryData = summaryQuery.data ?? null const [createSpecimenModalOpen, setCreateSpecimenModalOpen] = useState(false) @@ -67,8 +68,7 @@ export default function ControlBatchDetail() { if (!summaryData) return if (!window.confirm('Are you sure you want to delete this specimen and all its containers? This action cannot be undone.')) return try { - await controlsApi.deleteSpecimenFromBatch(summaryData.batch.id, specimenId) - refreshSummary() + await deleteSpecimen.mutateAsync({ batchId: summaryData.batch.id, specimenId }) } catch (err: any) { alert(err.response?.data?.error || 'Failed to delete specimen') } diff --git a/packages/web/src/pages/ReferenceData.tsx b/packages/web/src/pages/ReferenceData.tsx index b917eac2..b092fd07 100644 --- a/packages/web/src/pages/ReferenceData.tsx +++ b/packages/web/src/pages/ReferenceData.tsx @@ -10,9 +10,8 @@ import { type ReferenceDataConfig, type ReferenceDataType, } from '../config/reference-data-config' -import { useStorageTypes } from '../hooks/useReferenceData' +import { invalidateAfterReferenceDataWrite, useStorageTypes } from '../hooks/useReferenceData' import { - referenceDataPageKeys, useReferenceDataAllLocations, useReferenceDataTab, useSpecimenTypeContainerTypes, @@ -113,16 +112,6 @@ function ReferenceDataContent({ setPage(1) } - const refreshTabData = () => { - void queryClient.invalidateQueries({ - queryKey: referenceDataPageKeys.tab(activeTab, searchDebounced), - }) - } - - const refreshContainerTypeRelationships = () => { - void containerTypesQuery.refetch() - } - const handleToggleContainerType = async (specimenTypeId: number, containerType: string, isAdding: boolean) => { // Block non-admin users from toggling if (!isAdmin) { @@ -134,7 +123,7 @@ function ReferenceDataContent({ } else { await specimenTypesApi.removeContainerType(specimenTypeId, containerType) } - refreshContainerTypeRelationships() + void invalidateAfterReferenceDataWrite(queryClient) } useEffect(() => { @@ -155,10 +144,7 @@ function ReferenceDataContent({ } } setEditingItem(null) - refreshTabData() - if (activeTab === 'specimen-types') { - refreshContainerTypeRelationships() - } + void invalidateAfterReferenceDataWrite(queryClient) } catch (error: any) { throw error } @@ -167,7 +153,7 @@ function ReferenceDataContent({ const handleDelete = async (id: number) => { if (config.delete) { await config.delete(id) - refreshTabData() + void invalidateAfterReferenceDataWrite(queryClient, { type: activeTab, id }) } } diff --git a/packages/web/src/pages/SpecimenDetail.tsx b/packages/web/src/pages/SpecimenDetail.tsx index da56f1cd..d5770950 100644 --- a/packages/web/src/pages/SpecimenDetail.tsx +++ b/packages/web/src/pages/SpecimenDetail.tsx @@ -1,6 +1,5 @@ import { useEffect, useRef, useState } from 'react' import { useParams, Link, useNavigate, useSearchParams } from 'react-router-dom' -import { useQueryClient } from '@tanstack/react-query' import EntityBreadcrumbs from '../components/EntityBreadcrumbs' import AddContainerForSpecimenModal from '../components/AddContainerForSpecimenModal' import { getSpecimenTypeIcon, getContainerTypeIcon, getContainerTypeName } from '../lib/icons' @@ -10,7 +9,6 @@ import { useSpecimen, useContainersForSpecimen, useSpecimenSourceInfo, - specimenKeys, } from '../hooks/useSpecimens' import { Button, @@ -40,7 +38,6 @@ interface Container { export default function SpecimenDetail() { const { id } = useParams<{ id: string }>() const navigate = useNavigate() - const queryClient = useQueryClient() const { canWrite } = useUser() const specimenId = id != null ? parseInt(id, 10) : NaN const specimenQuery = useSpecimen(specimenId) @@ -459,9 +456,6 @@ export default function SpecimenDetail() { onClose={() => setAddContainerModalOpen(false)} specimenId={specimen.id} specimenTypeId={specimen.specimenTypeId} - onSuccess={() => { - void queryClient.invalidateQueries({ queryKey: specimenKeys.containers(id!) }) - }} />
diff --git a/packages/web/src/pages/StudyDetail.tsx b/packages/web/src/pages/StudyDetail.tsx index 02052e01..4e24b145 100644 --- a/packages/web/src/pages/StudyDetail.tsx +++ b/packages/web/src/pages/StudyDetail.tsx @@ -199,8 +199,8 @@ export default function StudyDetail() { subject.name.toLowerCase().includes(searchQuery.toLowerCase()) ) + // useMergeSubjects has already refreshed the queries the merge changed. const handleMergeSuccess = () => { - invalidateSubjects() setMergeModalOpen(false) } diff --git a/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx b/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx index 64edf5c0..91354920 100644 --- a/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx +++ b/packages/web/src/pages/__tests__/ControlBatchDetail.test.tsx @@ -1,6 +1,17 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { render, screen, waitFor } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' import ControlBatchDetail from '../ControlBatchDetail' +import { controlsApi } from '../../lib/api/controls' +import { collectionKeys } from '../../hooks/useCollections' +import { controlKeys } from '../../hooks/useControls' +import { specimenKeys } from '../../hooks/useSpecimens' +import { dashboardKeys } from '../../hooks/useDashboard' +import { + createInvalidationTestClient, + keysNotInvalidated, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' vi.mock('react-router-dom', async () => { const actual = await vi.importActual('react-router-dom') @@ -33,6 +44,7 @@ vi.mock('../../lib/api/controls', async () => { }, }), deleteBatch: vi.fn().mockResolvedValue(undefined), + deleteSpecimenFromBatch: vi.fn().mockResolvedValue({ message: 'Deleted' }), } }) }) @@ -71,4 +83,28 @@ describe('ControlBatchDetail', () => { expect(await screen.findByRole('button', { name: /^Edit$/ })).toBeInTheDocument() expect(screen.queryByRole('button', { name: /Delete Batch/i })).not.toBeInTheDocument() }) + + it('refreshes the Collection grids, Specimen lists and dashboard after deleting a Specimen from the batch', async () => { + const user = userEvent.setup() + vi.spyOn(window, 'confirm').mockReturnValue(true) + const loaded = await controlsApi.getBatchSummary(1) + vi.mocked(controlsApi.getBatchSummary).mockResolvedValueOnce({ + ...loaded, + specimens: [{ id: 30, specimenTypeName: 'Whole blood', collectionDate: '2024-01-02' }], + } as never) + const queryClient = createInvalidationTestClient() + const otherPages = [ + collectionKeys.micronixPlate(4), + controlKeys.overview(), + specimenKeys.list({}), + dashboardKeys.count('specimens'), + ] + seedQueries(queryClient, otherPages) + await render(, { queryClient }) + + await user.click(await screen.findByRole('button', { name: 'Delete' })) + + await waitFor(() => expect(controlsApi.deleteSpecimenFromBatch).toHaveBeenCalledWith(1, 30)) + await waitFor(() => expect(keysNotInvalidated(queryClient, otherPages)).toEqual([])) + }) }) diff --git a/packages/web/src/pages/__tests__/ReferenceData.test.tsx b/packages/web/src/pages/__tests__/ReferenceData.test.tsx index 6eb58680..bc555ea9 100644 --- a/packages/web/src/pages/__tests__/ReferenceData.test.tsx +++ b/packages/web/src/pages/__tests__/ReferenceData.test.tsx @@ -2,6 +2,16 @@ import { describe, it, expect, beforeEach, vi } from 'vitest' import { render, screen, waitFor } from '../../__tests__/helpers/render' import userEvent from '@testing-library/user-event' import ReferenceData from '../ReferenceData' +import { getReferenceDataConfig } from '../../config/reference-data-config' +import { specimenTypeKeys, tagKeys } from '../../hooks/useReferenceData' +import { referenceDataPageKeys } from '../../hooks/useReferenceDataPage' +import { exportWorkflowKeys } from '../../hooks/useExportWorkflow' +import { + createInvalidationTestClient, + keysNotInvalidated, + observeQuery, + seedQueries, +} from '../../__tests__/helpers/query-invalidation' const mockUseReferenceDataTab = vi.hoisted(() => vi.fn(() => ({ @@ -96,14 +106,20 @@ vi.mock('../../lib/api/locations', async () => { }) // Mock the hooks -vi.mock('../../hooks/useReferenceData', () => ({ - useStorageTypes: vi.fn().mockReturnValue({ - data: [], - isLoading: false, - isError: false, - error: null, - }), -})) +vi.mock('../../hooks/useReferenceData', async () => { + const actual = await vi.importActual( + '../../hooks/useReferenceData' + ) + return { + ...actual, + useStorageTypes: vi.fn().mockReturnValue({ + data: [], + isLoading: false, + isError: false, + error: null, + }), + } +}) // Mock the UserContext const mockUser = { @@ -230,6 +246,57 @@ describe('ReferenceData Page', () => { }) }) + describe('Refreshing other pages', () => { + /** Pages outside Reference Data that list Reference data: the export Tag filter and the Specimen form's specimen types. */ + const otherPages = [ + [...exportWorkflowKeys.referenceData(), 'tags'], + specimenTypeKeys.list(), + tagKeys.list(), + ] + + it('invalidates the export Tag filter and Specimen form after saving a new item', async () => { + const user = userEvent.setup() + const queryClient = createInvalidationTestClient() + seedQueries(queryClient, otherPages) + await render(, { queryClient, initialEntries: ['/reference-data?tab=specimen-types'] }) + + await user.click(screen.getByText('Add New')) + // The add form labels its submit button Update, since the page passes it an empty item. + await user.click(await screen.findByRole('button', { name: /^(Create|Update)$/ })) + + await waitFor(() => expect(getReferenceDataConfig('specimen-types')?.create).toHaveBeenCalled()) + await waitFor(() => expect(keysNotInvalidated(queryClient, otherPages)).toEqual([])) + }) + + it("invalidates other pages after a delete without refetching the deleted item's queries", async () => { + const user = userEvent.setup() + vi.spyOn(window, 'confirm').mockReturnValue(true) + mockUseReferenceDataTab.mockReturnValue({ + data: [{ id: 3, name: 'Plasma' }], + isPending: false, + isLoading: false, + isError: false, + isSuccess: true, + error: null, + refetch: vi.fn(), + } as unknown as ReturnType) + const queryClient = createInvalidationTestClient() + const tags = await observeQuery(queryClient, tagKeys.list()) + const deletedDetail = await observeQuery(queryClient, specimenTypeKeys.detail(3)) + const deletedContainerTypes = await observeQuery(queryClient, referenceDataPageKeys.containerTypes(3)) + await render(, { queryClient, initialEntries: ['/reference-data?tab=specimen-types'] }) + + await user.click(screen.getByRole('button', { name: 'Delete' })) + + await waitFor(() => expect(getReferenceDataConfig('specimen-types')?.delete).toHaveBeenCalledWith(3)) + await waitFor(() => expect(tags.queryFn).toHaveBeenCalledTimes(2)) + expect(deletedDetail.queryFn).toHaveBeenCalledTimes(1) + expect(deletedContainerTypes.queryFn).toHaveBeenCalledTimes(1) + expect(queryClient.getQueryState(specimenTypeKeys.detail(3))?.isInvalidated).toBe(true) + for (const observed of [tags, deletedDetail, deletedContainerTypes]) observed.unsubscribe() + }) + }) + describe('Locations Tab', () => { it('renders locations tab when clicked', async () => { const user = userEvent.setup() From 398d2adb9d431562b7bfba8e83159802461030df Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sat, 31 Oct 2026 23:20:27 -0700 Subject: [PATCH 31/54] fix(api): read pagination settings through the settings cache constants.ts kept its own module-level copy of the pagination settings that was never cleared, so a new page size applied only after a restart, and tests passed or failed depending on which test filled it first. validateLimit now reads through the settings cache, which is keyed per database and updated when settings are saved. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/__tests__/constants.test.ts | 47 +++++++++++++++++++ packages/api/src/lib/constants.ts | 13 ++--- 2 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 packages/api/src/lib/__tests__/constants.test.ts diff --git a/packages/api/src/lib/__tests__/constants.test.ts b/packages/api/src/lib/__tests__/constants.test.ts new file mode 100644 index 00000000..3a342de3 --- /dev/null +++ b/packages/api/src/lib/__tests__/constants.test.ts @@ -0,0 +1,47 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import type { Database } from '../../db/client' +import type { Database as SQLiteDatabase } from 'bun:sqlite' +import { setupTestDatabase, cleanupTestDatabase } from '../../__tests__/helpers/db-setup' +import { setupPaginationSettings } from '../../__tests__/helpers/auth-helpers' +import { setPaginationSettings } from '../settings' +import { validateLimit } from '../constants' + +describe('validateLimit', () => { + let db: Database + let sqlite: SQLiteDatabase + + beforeEach(async () => { + const setup = await setupTestDatabase() + db = setup.db + sqlite = setup.sqlite + }) + + afterEach(() => { + cleanupTestDatabase(sqlite) + }) + + it('uses a changed page size without a restart', async () => { + await setupPaginationSettings(db, { defaultPageSize: 25, maxPageSize: 100 }) + expect(await validateLimit(db, undefined)).toBe(25) + expect(await validateLimit(db, 80)).toBe(80) + + await setPaginationSettings(db, { defaultPageSize: 10, maxPageSize: 50 }) + + expect(await validateLimit(db, undefined)).toBe(10) + expect(await validateLimit(db, 80)).toBe(50) + }) + + it('reads each database’s own settings', async () => { + await setupPaginationSettings(db, { defaultPageSize: 25, maxPageSize: 100 }) + expect(await validateLimit(db, undefined)).toBe(25) + + const other = await setupTestDatabase() + try { + await expect(validateLimit(other.db, undefined)).rejects.toThrow('Pagination settings are not configured') + await setupPaginationSettings(other.db, { defaultPageSize: 7, maxPageSize: 20 }) + expect(await validateLimit(other.db, undefined)).toBe(7) + } finally { + cleanupTestDatabase(other.sqlite) + } + }) +}) diff --git a/packages/api/src/lib/constants.ts b/packages/api/src/lib/constants.ts index b32f95f7..e685285a 100644 --- a/packages/api/src/lib/constants.ts +++ b/packages/api/src/lib/constants.ts @@ -1,27 +1,20 @@ import type { Database } from '../db/client' import { pageQueryParam } from './query-params' -// Cache for pagination settings -let paginationCache: { defaultPageSize: number; maxPageSize: number } | null = null - /** - * Get pagination settings from cache or database + * Get pagination settings through the settings cache, which is keyed per database and + * cleared when an admin saves settings, so a new page size applies without a restart. * @param db - Database instance * @throws Error if pagination settings are not configured */ async function getPaginationSettings(db: Database): Promise<{ defaultPageSize: number; maxPageSize: number }> { - if (paginationCache) { - return paginationCache - } - const { getPaginationSettings } = await import('./settings') const settings = await getPaginationSettings(db) - + if (!settings) { throw new Error('Pagination settings are not configured. Please run database initialization.') } - paginationCache = settings return settings } From c12132c6ecaac8bb290e29fb11359644e555b8b2 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:12:45 -0800 Subject: [PATCH 32/54] fix(api): clear specimen type cache, dedupe sheet moves, auth the units read (#134) - Specimen type create, rename and delete clear the cached Specimen type list the dashboard statistics read, through a new `afterWrite` hook on createCrudRoutes. - moveSheetsToCollection deduplicates sheet ids, so a Sheet listed twice moves once and the response reports moved: 1. - GET /api/settings/container-types/:containerType/units requires a logged-in user, like the other settings reads. Co-Authored-By: Claude Opus 5.5 --- .../api/src/lib/collections/sheet-move.ts | 11 ++- packages/api/src/lib/crud-routes.ts | 9 +++ .../src/routes/__tests__/collections.test.ts | 22 ++++++ .../api/src/routes/__tests__/settings.test.ts | 26 +++++++ .../specimen-type-statistics.test.ts | 78 +++++++++++++++++++ packages/api/src/routes/settings.ts | 2 +- packages/api/src/routes/specimen-types.ts | 3 + 7 files changed, 146 insertions(+), 5 deletions(-) create mode 100644 packages/api/src/routes/__tests__/specimen-type-statistics.test.ts diff --git a/packages/api/src/lib/collections/sheet-move.ts b/packages/api/src/lib/collections/sheet-move.ts index 4a73d0fe..0909e2c8 100644 --- a/packages/api/src/lib/collections/sheet-move.ts +++ b/packages/api/src/lib/collections/sheet-move.ts @@ -31,6 +31,9 @@ export async function moveSheetsToCollection( targetCollectionId: number, targetCollectionType: 'box' | 'bag', ): Promise<{ moved: number }> { + // A Sheet listed twice moves once and counts once. + const uniqueSheetIds = [...new Set(sheetIds)] + // Checks and updates run synchronously inside the transaction: with no await between // them, a concurrent move cannot slip in, so its loser gets SheetNameConflictError // rather than the UNIQUE constraint error. @@ -44,8 +47,8 @@ export async function moveSheetsToCollection( if (targetCollectionType === 'box') { // Sheet names are unique within a box (bags still allow repeated legacy names). const moving = - sheetIds.length > 0 - ? tx.select({ id: sheet.id, name: sheet.name }).from(sheet).where(inArray(sheet.id, sheetIds)).all() + uniqueSheetIds.length > 0 + ? tx.select({ id: sheet.id, name: sheet.name }).from(sheet).where(inArray(sheet.id, uniqueSheetIds)).all() : [] const staying = tx .select({ id: sheet.id, name: sheet.name }) @@ -60,7 +63,7 @@ export async function moveSheetsToCollection( } } - for (const sheetId of sheetIds) { + for (const sheetId of uniqueSheetIds) { const updated = tx .update(sheet) .set({ @@ -77,5 +80,5 @@ export async function moveSheetsToCollection( } }) - return { moved: sheetIds.length } + return { moved: uniqueSheetIds.length } } diff --git a/packages/api/src/lib/crud-routes.ts b/packages/api/src/lib/crud-routes.ts index 1144dae2..a7a96572 100644 --- a/packages/api/src/lib/crud-routes.ts +++ b/packages/api/src/lib/crud-routes.ts @@ -94,6 +94,11 @@ export interface CrudRouteConfig< * Optional: Custom validation before update */ validateUpdate?: (id: number, data: TUpdate, database: Database) => Promise + + /** + * Optional: Run after a create, update or delete succeeds, such as to clear a cache of the table + */ + afterWrite?: () => void } /** @@ -125,6 +130,7 @@ export function createCrudRoutes< onUpdateDefaults, validateCreate, validateUpdate, + afterWrite, } = config const updateSchema = (providedUpdateSchema || createSchema) as z.ZodType @@ -232,6 +238,7 @@ export function createCrudRoutes< .returning() const row = insertedRow(result, entityName) + afterWrite?.() const transformed = transformDetail ? transformDetail(row as TSelect) : row return createdResponse(c, transformed) } catch (error) { @@ -311,6 +318,7 @@ export function createCrudRoutes< if (!row) { throw new NotFoundError(entityName, id) } + afterWrite?.() const transformed = transformDetail ? transformDetail(row as TSelect) : row return successResponse(c, transformed) } catch (error) { @@ -343,6 +351,7 @@ export function createCrudRoutes< if (result.length === 0) { throw new NotFoundError(entityName, id) } + afterWrite?.() return c.json({ message: `${entityName} deleted successfully` }) } catch (error) { diff --git a/packages/api/src/routes/__tests__/collections.test.ts b/packages/api/src/routes/__tests__/collections.test.ts index c1a6d57a..52c9afa0 100644 --- a/packages/api/src/routes/__tests__/collections.test.ts +++ b/packages/api/src/routes/__tests__/collections.test.ts @@ -496,6 +496,28 @@ describe('Collections API', () => { }) describe('POST /api/collections/sheets/move', () => { + it('moves a Sheet listed twice once and reports moved: 1', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const loc = await createTestLocation(ctx.db, { + name: 'Loc', + storageTypeId: String(storageType.id), + canContainCollections: true, + }) + const boxA = await createTestBox(ctx.db, { name: 'BoxA', locationId: loc.id }) + const boxB = await createTestBox(ctx.db, { name: 'BoxB', locationId: loc.id }) + const moving = await createTestSheet(ctx.db, { name: 'Sheet-1', boxId: boxA.id }) + + const res = await ctx.request('/api/collections/sheets/move', { + method: 'POST', + json: { sheetIds: [moving.id, moving.id], targetCollectionId: boxB.id, targetCollectionType: 'box' }, + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ success: true, moved: 1 }) + const moved = await ctx.db.select().from(sheet).where(eq(sheet.id, moving.id)).get() + expect(moved?.boxId).toBe(boxB.id) + }) + it('returns a typed 409 when the target box already has a sheet with that name', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) const loc = await createTestLocation(ctx.db, { diff --git a/packages/api/src/routes/__tests__/settings.test.ts b/packages/api/src/routes/__tests__/settings.test.ts index b967436b..05977a57 100644 --- a/packages/api/src/routes/__tests__/settings.test.ts +++ b/packages/api/src/routes/__tests__/settings.test.ts @@ -4,6 +4,7 @@ import { setupAuthenticatedRouteTest, type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' +import { createTestContainerTypeUnit, createTestUnit } from '../../__tests__/helpers/factories' import { createSettingsRoutes } from '../settings' describe('Settings API', () => { @@ -17,6 +18,7 @@ describe('Settings API', () => { password: 'password123', role: 'admin', }, + additionalUsers: [{ key: 'viewer', email: 'viewer@test.com', password: 'password123', role: 'viewer' }], mount: (app, { db }) => { app.route('/api/settings', createSettingsRoutes(db)) }, @@ -46,6 +48,30 @@ describe('Settings API', () => { }) }) + describe('GET /api/settings/container-types/:containerType/units', () => { + it('returns 401 when not authenticated', async () => { + const res = await authenticatedRequest(ctx.createRequestApp(), '/api/settings/container-types/paper/units', { + method: 'GET', + }) + expect(res.status).toBe(401) + }) + + it("returns the container type's units to a viewer", async () => { + const microliter = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await createTestContainerTypeUnit(ctx.db, { containerType: 'cryovial_tube', unitId: microliter.id }) + + const res = await ctx.request('/api/settings/container-types/cryovial_tube/units', { + method: 'GET', + cookie: ctx.cookies.viewer, + }) + + expect(res.status).toBe(200) + expect(await res.json()).toEqual({ + units: [{ id: microliter.id, symbol: 'uL', name: 'microliter', category: 'volume' }], + }) + }) + }) + describe('PUT /api/settings/:key', () => { for (const userId of ['abc', -1, 1.5]) { it(`rejects userId ${JSON.stringify(userId)} with 400`, async () => { diff --git a/packages/api/src/routes/__tests__/specimen-type-statistics.test.ts b/packages/api/src/routes/__tests__/specimen-type-statistics.test.ts new file mode 100644 index 00000000..330ca328 --- /dev/null +++ b/packages/api/src/routes/__tests__/specimen-type-statistics.test.ts @@ -0,0 +1,78 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { + setupAuthenticatedRouteTest, + type AuthenticatedRouteTestContext, +} from '../../__tests__/helpers/authenticated-route-test' +import { createTestSpecimen, createTestSpecimenType } from '../../__tests__/helpers/factories' +import { cache } from '../../lib/cache' +import { createSpecimenTypesRoutes } from '../specimen-types' +import { createStatisticsRoutes } from '../statistics' + +// The dashboard statistics cache the Specimen type list; writes in Reference Data must refresh it. +describe('Specimen type writes and dashboard statistics', () => { + let ctx: AuthenticatedRouteTestContext + + beforeEach(async () => { + // The cache is process-wide, so drop whatever an earlier test's database left in it. + cache.clear() + ctx = await setupAuthenticatedRouteTest({ + user: { email: 'admin@test.com', name: 'Admin', password: 'password123', role: 'admin' }, + mount: (app, { db }) => { + app.route('/api/specimen-types', createSpecimenTypesRoutes(db)) + app.route('/api/statistics', createStatisticsRoutes(db)) + }, + }) + }) + + afterEach(() => { + ctx.cleanup() + cache.clear() + }) + + async function bySpecimenType(): Promise> { + const res = await ctx.request('/api/statistics', { method: 'GET' }) + expect(res.status).toBe(200) + return ((await res.json()) as { specimens: { bySpecimenType: Record } }).specimens.bySpecimenType + } + + it('shows a Specimen type added after the statistics were cached', async () => { + const blood = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + await createTestSpecimen(ctx.db, blood.id) + expect(await bySpecimenType()).toEqual({ Blood: 1 }) + + const res = await ctx.request('/api/specimen-types', { method: 'POST', json: { name: 'Saliva' } }) + expect(res.status).toBe(201) + const { data: saliva } = (await res.json()) as { data: { id: number } } + await createTestSpecimen(ctx.db, saliva.id) + + expect(await bySpecimenType()).toEqual({ Blood: 1, Saliva: 1 }) + }) + + it('shows the new name of a renamed Specimen type', async () => { + const blood = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + await createTestSpecimen(ctx.db, blood.id) + expect(await bySpecimenType()).toEqual({ Blood: 1 }) + + const res = await ctx.request(`/api/specimen-types/${blood.id}`, { method: 'PUT', json: { name: 'Whole blood' } }) + expect(res.status).toBe(200) + + expect(await bySpecimenType()).toEqual({ 'Whole blood': 1 }) + }) + + it('forgets a deleted Specimen type', async () => { + const blood = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const unused = await createTestSpecimenType(ctx.db, { name: 'Unused' }) + await createTestSpecimen(ctx.db, blood.id) + expect(await bySpecimenType()).toEqual({ Blood: 1 }) + + const res = await ctx.request(`/api/specimen-types/${unused.id}`, { method: 'DELETE' }) + expect(res.status).toBe(200) + + // SQLite reuses the highest rowid, so a type added now takes the deleted type's id. + const added = await createTestSpecimenType(ctx.db, { name: 'Saliva' }) + expect(added.id).toBe(unused.id) + await createTestSpecimen(ctx.db, added.id) + + expect(await bySpecimenType()).toEqual({ Blood: 1, Saliva: 1 }) + }) +}) diff --git a/packages/api/src/routes/settings.ts b/packages/api/src/routes/settings.ts index 0a3b1a30..ed8d725c 100644 --- a/packages/api/src/routes/settings.ts +++ b/packages/api/src/routes/settings.ts @@ -526,7 +526,7 @@ const tableViewConfigurationsSchema = z.object({ const containerTypeSchema = z.enum(['paper', 'cryovial_tube', 'micronix_tube', 'static_well']) // GET /api/settings/container-types/:containerType/units - Get allowed units for a container type - settings.get('/container-types/:containerType/units', async (c) => { + settings.get('/container-types/:containerType/units', authMiddleware, async (c) => { try { const containerType = requireParam(c, 'containerType') diff --git a/packages/api/src/routes/specimen-types.ts b/packages/api/src/routes/specimen-types.ts index d49d55f6..d58f98f8 100644 --- a/packages/api/src/routes/specimen-types.ts +++ b/packages/api/src/routes/specimen-types.ts @@ -1,4 +1,5 @@ import { Hono } from 'hono' +import { cache, cacheKeys } from '../lib/cache' import { createCrudRoutes } from '../lib/crud-routes' import type { Database } from '../db/client' import { @@ -188,6 +189,8 @@ export function createSpecimenTypesRoutes(database: Database): Hono { checkInUse: checkSpecimenTypeInUse, onCreateDefaults, onUpdateDefaults, + // The dashboard statistics cache the Specimen type list (lib/statistics/specimen-aggregates.ts). + afterWrite: () => cache.delete(cacheKeys.specimenTypes), }) // Additional routes for container type relationships From 8367a1264c088c8f278d3e6e90b134afa342e631 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:13:01 -0800 Subject: [PATCH 33/54] fix(db): back up the database before startup migrations (#132) When a migration is pending on an existing file, openOperationalDatabase now writes `.pre-v-.bak` next to it with VACUUM INTO (so WAL pages are included) before the first migration or the legacy baseline runs. If the copy fails, startup throws MigrationBackupError and nothing is migrated. No copy is written when nothing is pending, for a database that was empty, for :memory:, or under NODE_ENV=test unless the caller passes backupBeforeMigrating: true. The db README and the deployment and backup guides describe the copy and when to delete it. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/db/README.md | 21 ++- .../src/db/__tests__/migration-backup.test.ts | 166 ++++++++++++++++++ packages/api/src/db/migration-backup.ts | 49 ++++++ packages/api/src/db/open.ts | 15 +- packages/api/src/db/schema-evolution.ts | 57 ++++-- .../guides/advanced/backup-and-restore.md | 18 ++ .../docs/guides/advanced/deployment.md | 2 + 7 files changed, 306 insertions(+), 22 deletions(-) create mode 100644 packages/api/src/db/__tests__/migration-backup.test.ts create mode 100644 packages/api/src/db/migration-backup.ts diff --git a/packages/api/src/db/README.md b/packages/api/src/db/README.md index b40792b2..9e17d1b1 100644 --- a/packages/api/src/db/README.md +++ b/packages/api/src/db/README.md @@ -32,7 +32,7 @@ Migration `001` is **only** for legacy files; fresh installs from the snapshot s ## Opening a database -Use `openOperationalDatabase(optionalPath?)` from the client module. It resolves the file path, enables WAL, turns on foreign keys, runs schema evolution, and returns Drizzle + raw `bun:sqlite` handles. Importing the client module does **not** open a connection. +Use `openOperationalDatabase(optionalPath?)` from the client module. It resolves the file path, enables WAL, turns on foreign keys, runs schema evolution (backing up the file first when a migration is pending, see Failures), and returns Drizzle + raw `bun:sqlite` handles. Importing the client module does **not** open a connection. The rules for transactions, DDL, and which entry point to open a database with are under Database in `CODING_STANDARDS.md`. @@ -66,4 +66,21 @@ When you add a multi-statement write, use `withWriteTransaction` instead of addi ## Failures -Each migration runs in a transaction. On failure, the transaction rolls back, `schema_version` is unchanged, and startup throws `SchemaMigrationError` with the migration number. Restore from backup or fix the database manually; there are no down-migrations. +Each migration runs in a transaction. On failure, the transaction rolls back, `schema_version` is unchanged, and startup throws `SchemaMigrationError` with the migration number. There are no down-migrations, so a migration that succeeds but leaves bad data can only be undone from a copy. + +### Backup before migrating + +Before the first migration changes an existing file (a numbered migration, or the legacy baseline `001`), `openOperationalDatabase` writes a copy next to it with `VACUUM INTO`, which includes pages still in the WAL: + +``` +.pre-v-.bak +e.g. sampledb.sqlite.pre-v11-20261101T143005123Z.bak +``` + +The version in the name is the one the migrations move to; the copy itself holds the version it had before. `migration-backup.ts` writes it, and `evolveOperationalSchema` calls it when passed `{ backupBeforeMigrating: true }`. + +- No copy when nothing is pending, when the file was empty (the snapshot path), or for `:memory:`. +- `openOperationalDatabase` skips the copy under `NODE_ENV=test`; pass `{ backupBeforeMigrating: true }` to force it. +- If the copy can't be written (read-only directory, full disk), startup throws `MigrationBackupError` before any migration runs, and the database is unchanged. + +Copies are never deleted automatically. The deployment guide in `packages/docs` says when to remove them. diff --git a/packages/api/src/db/__tests__/migration-backup.test.ts b/packages/api/src/db/__tests__/migration-backup.test.ts new file mode 100644 index 00000000..175ff995 --- /dev/null +++ b/packages/api/src/db/__tests__/migration-backup.test.ts @@ -0,0 +1,166 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Database } from 'bun:sqlite' +import { chmodSync, mkdtempSync, readdirSync, rmSync } from 'fs' +import { join } from 'path' +import { tmpdir } from 'os' +import { CURRENT_SCHEMA_VERSION, evolveOperationalSchema, getRecordedSchemaVersion } from '../schema-evolution' +import { openOperationalDatabase } from '../open' +import { MigrationBackupError } from '../migration-backup' + +const PREVIOUS_VERSION = CURRENT_SCHEMA_VERSION - 1 + +/** A database one version behind current, holding three studies. */ +function createDatabaseOneVersionBehind(path: string): void { + const sqlite = new Database(path) + evolveOperationalSchema(sqlite) + for (const code of ['A', 'B', 'C']) { + sqlite + .prepare( + `INSERT INTO study (title, short_code, is_longitudinal, lead_person, created, last_updated) + VALUES (?, ?, 0, 'Lead', '2026-01-01', '2026-01-01')`, + ) + .run(`Study ${code}`, code) + } + sqlite.prepare('UPDATE schema_version SET version = ?').run(PREVIOUS_VERSION) + sqlite.close() +} + +function backupFiles(dir: string): string[] { + return readdirSync(dir).filter((name) => name.endsWith('.bak')) +} + +describe('backup before startup migrations', () => { + let dir: string + let dbPath: string + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'migration-backup-')) + dbPath = join(dir, 'sampledb.sqlite') + }) + + afterEach(() => { + chmodSync(dir, 0o755) + rmSync(dir, { recursive: true, force: true }) + }) + + it('writes a copy with the pre-migration version and rows before migrating', () => { + createDatabaseOneVersionBehind(dbPath) + + const { sqlite } = openOperationalDatabase(dbPath, { backupBeforeMigrating: true }) + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + sqlite.close() + + const backups = backupFiles(dir) + expect(backups).toHaveLength(1) + expect(backups[0]).toMatch( + new RegExp(`^sampledb\\.sqlite\\.pre-v${CURRENT_SCHEMA_VERSION}-\\d{8}T\\d{6}\\d{3}Z\\.bak$`), + ) + + // Read-only: inspecting the copy must not migrate it. + const copy = new Database(join(dir, backups[0]), { readonly: true }) + expect(getRecordedSchemaVersion(copy)).toBe(PREVIOUS_VERSION) + expect(copy.prepare('SELECT COUNT(*) AS n FROM study').get()).toEqual({ n: 3 }) + copy.close() + }) + + it('includes rows another connection left in the WAL', () => { + createDatabaseOneVersionBehind(dbPath) + const writer = new Database(dbPath) + writer.exec('PRAGMA journal_mode = WAL') + writer.exec('PRAGMA wal_autocheckpoint = 0') + writer + .prepare( + `INSERT INTO study (title, short_code, is_longitudinal, lead_person, created, last_updated) + VALUES ('Study D', 'D', 0, 'Lead', '2026-01-01', '2026-01-01')`, + ) + .run() + + const { sqlite } = openOperationalDatabase(dbPath, { backupBeforeMigrating: true }) + sqlite.close() + writer.close() + + const [backup] = backupFiles(dir) + const copy = new Database(join(dir, backup), { readonly: true }) + expect(copy.prepare('SELECT COUNT(*) AS n FROM study').get()).toEqual({ n: 4 }) + copy.close() + }) + + it('backs up an unversioned legacy database before its baseline migration', () => { + const legacy = new Database(dbPath) + legacy.exec('CREATE TABLE study (id INTEGER PRIMARY KEY, title TEXT, short_code TEXT)') + legacy.exec("INSERT INTO study (title, short_code) VALUES ('Legacy', 'L')") + legacy.close() + + // Only the copy matters here; the bare legacy file can't finish migrating. + try { + openOperationalDatabase(dbPath, { backupBeforeMigrating: true }).sqlite.close() + } catch { + // ignore + } + + const [backup] = backupFiles(dir) + const copy = new Database(join(dir, backup), { readonly: true }) + expect(getRecordedSchemaVersion(copy)).toBeNull() + expect(copy.prepare('SELECT COUNT(*) AS n FROM study').get()).toEqual({ n: 1 }) + copy.close() + }) + + it('writes no copy when no migration is pending', () => { + openOperationalDatabase(dbPath, { backupBeforeMigrating: true }).sqlite.close() + openOperationalDatabase(dbPath, { backupBeforeMigrating: true }).sqlite.close() + + expect(backupFiles(dir)).toEqual([]) + }) + + it('writes no copy of a test database by default', () => { + expect(process.env.NODE_ENV).toBe('test') + createDatabaseOneVersionBehind(dbPath) + + const { sqlite } = openOperationalDatabase(dbPath) + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + sqlite.close() + + expect(backupFiles(dir)).toEqual([]) + }) + + it('migrates an in-memory database without a copy', () => { + // A copy of ':memory:' would land relative to the working directory, so run from `dir`. + const previousCwd = process.cwd() + process.chdir(dir) + try { + const { sqlite } = openOperationalDatabase(':memory:', { backupBeforeMigrating: true }) + sqlite.prepare('UPDATE schema_version SET version = ?').run(PREVIOUS_VERSION) + + evolveOperationalSchema(sqlite, { backupBeforeMigrating: true }) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + sqlite.close() + } finally { + process.chdir(previousCwd) + } + expect(backupFiles(dir)).toEqual([]) + }) + + it('stops without migrating when the copy cannot be written', () => { + createDatabaseOneVersionBehind(dbPath) + const sqlite = new Database(dbPath) + sqlite.exec('PRAGMA journal_mode = WAL') + expect(getRecordedSchemaVersion(sqlite)).toBe(PREVIOUS_VERSION) + chmodSync(dir, 0o555) + + let thrown: unknown + try { + evolveOperationalSchema(sqlite, { backupBeforeMigrating: true }) + } catch (error: unknown) { + thrown = error + } + + expect(thrown).toBeInstanceOf(MigrationBackupError) + expect((thrown as Error).message).toContain(`Could not back up ${dbPath}`) + expect((thrown as Error).message).toContain('No migrations were run') + expect(getRecordedSchemaVersion(sqlite)).toBe(PREVIOUS_VERSION) + sqlite.close() + chmodSync(dir, 0o755) + expect(backupFiles(dir)).toEqual([]) + }) +}) diff --git a/packages/api/src/db/migration-backup.ts b/packages/api/src/db/migration-backup.ts new file mode 100644 index 00000000..e98f5b51 --- /dev/null +++ b/packages/api/src/db/migration-backup.ts @@ -0,0 +1,49 @@ +import { Database } from 'bun:sqlite' + +export class MigrationBackupError extends Error { + readonly databasePath: string + readonly backupPath: string + + constructor(databasePath: string, backupPath: string, cause: unknown) { + const detail = cause instanceof Error ? cause.message : String(cause) + super( + `Could not back up ${databasePath} to ${backupPath} before running schema migrations: ${detail}. ` + + 'No migrations were run and the database is unchanged. Make the directory writable ' + + '(and check free space), then start again.', + ) + this.name = 'MigrationBackupError' + this.databasePath = databasePath + this.backupPath = backupPath + if (cause instanceof Error) { + this.cause = cause + } + } +} + +/** `20261101T143005123Z`: UTC, sortable, and safe in a file name. */ +function backupTimestamp(now: Date): string { + return now.toISOString().replace(/[-:.]/g, '') +} + +/** + * Write a consistent copy of the database next to its file before migrations run, so a + * migration that goes wrong can be undone without falling back to the last restic snapshot. + * `VACUUM INTO` reads through the connection, so the copy includes pages still in the WAL. + * + * Returns the copy's path, or null for an in-memory database (there is no file to protect). + * Throws MigrationBackupError when the copy can't be written; the caller must not migrate. + */ +export function backupDatabaseBeforeMigrating(sqlite: Database, targetVersion: number): string | null { + const databasePath = sqlite.filename + if (databasePath === '' || databasePath === ':memory:') { + return null + } + const backupPath = `${databasePath}.pre-v${targetVersion}-${backupTimestamp(new Date())}.bak` + try { + sqlite.prepare('VACUUM INTO ?').run(backupPath) + } catch (error: unknown) { + throw new MigrationBackupError(databasePath, backupPath, error) + } + console.log(`💾 Backed up database before schema migration: ${backupPath}`) + return backupPath +} diff --git a/packages/api/src/db/open.ts b/packages/api/src/db/open.ts index 6e79c029..83cd9df1 100644 --- a/packages/api/src/db/open.ts +++ b/packages/api/src/db/open.ts @@ -4,7 +4,7 @@ import { existsSync, readFileSync } from 'fs' import { dirname, isAbsolute, join, resolve } from 'path' import { fileURLToPath } from 'url' import * as schema from './schema' -import { evolveOperationalSchema } from './schema-evolution' +import { evolveOperationalSchema, type EvolveOperationalSchemaOptions } from './schema-evolution' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -91,8 +91,15 @@ export type OperationalDatabase = ReturnType> /** * Open the operational SQLite database, evolve schema, return Drizzle + raw handles. + * + * Before migrating an existing file it writes a backup copy next to it, except under + * NODE_ENV=test, where every test database is disposable. Pass `backupBeforeMigrating` to + * override that default. */ -export function openOperationalDatabase(dbPath?: string): { +export function openOperationalDatabase( + dbPath?: string, + options: EvolveOperationalSchemaOptions = {}, +): { db: OperationalDatabase sqlite: SQLiteDatabase } { @@ -111,7 +118,9 @@ export function openOperationalDatabase(dbPath?: string): { sqlite.exec('PRAGMA foreign_keys = ON') try { - evolveOperationalSchema(sqlite) + evolveOperationalSchema(sqlite, { + backupBeforeMigrating: options.backupBeforeMigrating ?? process.env.NODE_ENV !== 'test', + }) } catch (error: unknown) { const message = error instanceof Error ? error.message : String(error) console.error(`❌ Error preparing database: ${message}`) diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index 0cbaaf71..e3903996 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -1,6 +1,7 @@ import { Database } from 'bun:sqlite' import { applyInitialSchema } from './apply-initial-schema' import { applyLegacyBaselineMigration } from './legacy-baseline' +import { backupDatabaseBeforeMigrating } from './migration-backup' import { ensureLegacyPaperTableForMigration003 } from './migration-preflight' import { listNumberedMigrations, runSqlMigration } from './migration-runner' @@ -42,22 +43,24 @@ export function isEmptyOperationalDatabase(sqlite: Database): boolean { return row.count === 0 } -function applyPendingMigrations(sqlite: Database, fromVersion: number): void { +function listPendingMigrations(fromVersion: number): Array<{ version: number; path: string; basename: string }> { if (fromVersion >= CURRENT_SCHEMA_VERSION) { - return + return [] } + return listNumberedMigrations().filter( + (migration) => + migration.version > fromVersion && + migration.version <= CURRENT_SCHEMA_VERSION && + // 001 runs only via applyLegacyBaselineMigration for unversioned databases. + migration.version !== LEGACY_BASELINE_VERSION, + ) +} - for (const migration of listNumberedMigrations()) { - if (migration.version <= fromVersion) { - continue - } - if (migration.version > CURRENT_SCHEMA_VERSION) { - continue - } - // 001 runs only via applyLegacyBaselineMigration for unversioned databases. - if (migration.version === LEGACY_BASELINE_VERSION) { - continue - } +function applyPendingMigrations( + sqlite: Database, + pending: Array<{ version: number; path: string; basename: string }>, +): void { + for (const migration of pending) { if (migration.version === PAPER_SUBLABEL_MIGRATION_VERSION) { ensureLegacyPaperTableForMigration003(sqlite) } @@ -68,10 +71,24 @@ function applyPendingMigrations(sqlite: Database, fromVersion: number): void { } } +export interface EvolveOperationalSchemaOptions { + /** + * Copy the database file next to itself (`VACUUM INTO`) before the first migration changes + * an existing database. Skipped when nothing is pending, for an empty database, and for an + * in-memory one. If the copy fails, nothing is migrated. + */ + backupBeforeMigrating?: boolean +} + /** * Bring an operational SQLite file to CURRENT_SCHEMA_VERSION. */ -export function evolveOperationalSchema(sqlite: Database): void { +export function evolveOperationalSchema(sqlite: Database, options: EvolveOperationalSchemaOptions = {}): void { + const backupIfRequested = (): void => { + if (options.backupBeforeMigrating) { + backupDatabaseBeforeMigrating(sqlite, CURRENT_SCHEMA_VERSION) + } + } const recorded = getRecordedSchemaVersion(sqlite) if (recorded != null) { @@ -80,7 +97,11 @@ export function evolveOperationalSchema(sqlite: Database): void { `Database schema version ${recorded} is newer than this application supports (${CURRENT_SCHEMA_VERSION}).`, ) } - applyPendingMigrations(sqlite, recorded) + const pending = listPendingMigrations(recorded) + if (pending.length > 0) { + backupIfRequested() + } + applyPendingMigrations(sqlite, pending) return } @@ -91,7 +112,8 @@ export function evolveOperationalSchema(sqlite: Database): void { applyInitialSchema(sqlite) const version = getRecordedSchemaVersion(sqlite) if (version == null || version < CURRENT_SCHEMA_VERSION) { - applyPendingMigrations(sqlite, version ?? 0) + // A database that was empty a moment ago has nothing to back up. + applyPendingMigrations(sqlite, listPendingMigrations(version ?? 0)) } const after = getRecordedSchemaVersion(sqlite) if (after !== CURRENT_SCHEMA_VERSION) { @@ -105,10 +127,11 @@ export function evolveOperationalSchema(sqlite: Database): void { return } + backupIfRequested() applyLegacyBaselineMigration(sqlite) const afterLegacy = getRecordedSchemaVersion(sqlite) if (afterLegacy == null) { throw new Error('Legacy baseline migration did not set schema_version') } - applyPendingMigrations(sqlite, afterLegacy) + applyPendingMigrations(sqlite, listPendingMigrations(afterLegacy)) } diff --git a/packages/docs/src/content/docs/guides/advanced/backup-and-restore.md b/packages/docs/src/content/docs/guides/advanced/backup-and-restore.md index 9219a25f..4f4f50cb 100644 --- a/packages/docs/src/content/docs/guides/advanced/backup-and-restore.md +++ b/packages/docs/src/content/docs/guides/advanced/backup-and-restore.md @@ -120,6 +120,24 @@ fly ssh console -a YOUR_APP -C 'f=$(mktemp) && sqlite3 /data/sampledb.sqlite ".b For lower RPO on Fly, consider **Litestream** (or similar) to object storage in addition to periodic restic snapshots. +## Copy before schema migrations + +Restic snapshots run on a schedule, so the newest one may be hours old when you upgrade. To cover upgrades, SampleDB copies the database itself when it starts on a database whose schema is older than the new version, before it changes anything: + +- **Where:** next to the database file, named `.pre-v-.bak`, for example `/var/lib/sampledb/sampledb.sqlite.pre-v11-20261101T143005123Z.bak` (Docker bind mount) or `/data/sampledb.sqlite.pre-v11-….bak` (fly.io). `` is the schema version the upgrade moves to; the copy holds the version from before the upgrade. +- **When:** only when a migration is pending. Restarting on the same version writes nothing. The startup log prints the path. +- **Space:** the copy is about the size of the database, so the directory needs free space for it. +- **If it fails:** startup stops with `Could not back up … before running schema migrations` and leaves the database unchanged. Make the directory writable or free space, then start again. + +SampleDB never deletes these copies. Keep the newest one until you've checked the upgraded app (log in, open a recent study and its containers) and the next restic backup has run, then delete it: + +```bash +ls -lh /var/lib/sampledb/*.bak +rm /var/lib/sampledb/sampledb.sqlite.pre-v11-20261101T143005123Z.bak +``` + +To undo an upgrade with the copy, stop SampleDB, deploy the previous image (the new one would migrate the copy again), and follow step 5 below, using the `.bak` file in place of the restic restore. A restic backup taken while the copy exists may include it if your script backs up the whole directory; `ops/backup/backup-db-restic.sh` backs up only `DATABASE_PATH`. + ## Restore runbook Use this when recovering from corruption, bad migration, or lost host. diff --git a/packages/docs/src/content/docs/guides/advanced/deployment.md b/packages/docs/src/content/docs/guides/advanced/deployment.md index ff615c7a..26238c13 100644 --- a/packages/docs/src/content/docs/guides/advanced/deployment.md +++ b/packages/docs/src/content/docs/guides/advanced/deployment.md @@ -48,6 +48,8 @@ The database lives at `$HOST_DATA_DIR/sampledb.sqlite` on the host, so your back **After deploys:** the browser UI is notified when a new build is on the server via `GET /api/app-version` (and `X-App-Build-Id`). Long‑lived tabs re-check on a timer and when the tab is focused, so users are prompted to refresh instead of running an old client against a new API. Local development uses a default `local-dev` id so the banner does not appear during `vite` dev. +**Upgrades and schema migrations:** when a new image starts on a database with an older schema, it first copies the database next to it (for example `$HOST_DATA_DIR/sampledb.sqlite.pre-v11-20261101T143005123Z.bak`), then migrates. If the copy can't be written, the app stops without migrating. Copies are not deleted automatically; see [Copy before schema migrations](/docs/guides/advanced/backup-and-restore/#copy-before-schema-migrations) for when to remove them. + For existing databases created before the error-logging feature, the `error_logs` table is created automatically on startup when missing. No manual migration is needed. ### Seeding a demo database (Docker) From 065d547511fae6a5f39b7151d02fa45f2d3d1712 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:15:07 -0800 Subject: [PATCH 34/54] fix(web): show Admin Users and Reference Data errors where the admin is looking (#137) Admin Users: - Create, edit, delete, reset-password and session errors now show inside the open modal (role="alert") instead of on the page behind it. Approve and restore, which have no modal, still use the page banner. - Modal submit buttons are disabled while a request is pending, and a ref guard ignores a second click in the same tick, so a double click sends one request. Cancel and the modal close are disabled while pending. - Modal labels are tied to their inputs (htmlFor/id). Reference Data: - A failed container-type toggle shows an error toast naming the type, and the page refetches container types on failure so the toggle shows the saved state. Toggles expose aria-pressed. - The add dialog's submit button says "Add" (it said "Update" because the page opens it with an empty item); the edit dialog still says "Update". Co-Authored-By: Claude Opus 5.5 --- .../src/components/ContainerTypeToggle.tsx | 9 +- .../web/src/components/ReferenceDataForm.tsx | 2 +- .../__tests__/ContainerTypeToggle.test.tsx | 67 ++++ .../__tests__/ReferenceDataForm.test.tsx | 35 ++- packages/web/src/pages/AdminUsers.tsx | 291 ++++++++++-------- packages/web/src/pages/ReferenceData.tsx | 14 +- .../src/pages/__tests__/AdminUsers.test.tsx | 181 ++++++++++- .../pages/__tests__/ReferenceData.test.tsx | 47 ++- 8 files changed, 503 insertions(+), 143 deletions(-) create mode 100644 packages/web/src/components/__tests__/ContainerTypeToggle.test.tsx diff --git a/packages/web/src/components/ContainerTypeToggle.tsx b/packages/web/src/components/ContainerTypeToggle.tsx index 134c9a40..b1f13e2d 100644 --- a/packages/web/src/components/ContainerTypeToggle.tsx +++ b/packages/web/src/components/ContainerTypeToggle.tsx @@ -1,5 +1,7 @@ import { useState } from 'react' import { CONTAINER_TYPES } from '../lib/container-types' +import { useToast } from '../contexts/ToastContext' +import { getQueryErrorMessage } from '../ui' interface ContainerTypeToggleProps { specimenTypeId: number @@ -17,6 +19,7 @@ export default function ContainerTypeToggle({ usageInfo = {}, }: ContainerTypeToggleProps) { const [loadingTypes, setLoadingTypes] = useState>(new Set()) + const { error: showError } = useToast() const handleToggle = async (containerType: string) => { if (disabled || loadingTypes.has(containerType)) return @@ -32,8 +35,9 @@ export default function ContainerTypeToggle({ try { await onToggle(specimenTypeId, containerType, !isSelected) } catch (error) { - console.error('Failed to toggle container type:', error) - // Error handling is done by parent component + const label = CONTAINER_TYPES.find((ct) => ct.value === containerType)?.label ?? containerType + const action = isSelected ? `remove ${label}` : `add ${label}` + showError(`Could not ${action}: ${getQueryErrorMessage(error, 'request failed')}`) } finally { setLoadingTypes((prev) => { const next = new Set(prev) @@ -66,6 +70,7 @@ export default function ContainerTypeToggle({ type="button" onClick={() => handleToggle(ct.value)} disabled={isDisabled} + aria-pressed={isSelected} className={`inline-flex items-center px-2 py-0.5 rounded text-xs font-medium transition-colors ${ isSelected ? isInUse diff --git a/packages/web/src/components/ReferenceDataForm.tsx b/packages/web/src/components/ReferenceDataForm.tsx index 6e40944e..98d5b439 100644 --- a/packages/web/src/components/ReferenceDataForm.tsx +++ b/packages/web/src/components/ReferenceDataForm.tsx @@ -311,7 +311,7 @@ export default function ReferenceDataForm({ disabled={loading} className="px-4 py-2 bg-app-accent text-white rounded-lg hover:bg-app-accent-hover disabled:opacity-50" > - {loading ? 'Saving...' : item ? 'Update' : 'Create'} + {loading ? 'Saving...' : item?.id != null ? 'Update' : 'Add'} diff --git a/packages/web/src/components/__tests__/ContainerTypeToggle.test.tsx b/packages/web/src/components/__tests__/ContainerTypeToggle.test.tsx new file mode 100644 index 00000000..f153674d --- /dev/null +++ b/packages/web/src/components/__tests__/ContainerTypeToggle.test.tsx @@ -0,0 +1,67 @@ +import { describe, it, expect, vi } from 'vitest' +import { render, screen, waitFor } from '../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' +import ContainerTypeToggle from '../ContainerTypeToggle' +import { ToastContainer } from '../Toast' + +function apiError(status: number, error: string) { + return Object.assign(new Error(`Request failed with status code ${status}`), { + response: { status, data: { error } }, + }) +} + +describe('ContainerTypeToggle', () => { + it('marks each allowed container type as pressed', async () => { + await render() + + expect(screen.getByRole('button', { name: /cryovial/i })).toHaveAttribute('aria-pressed', 'true') + expect(screen.getByRole('button', { name: /micronix/i })).toHaveAttribute('aria-pressed', 'false') + }) + + it('turns a container type on through onToggle', async () => { + const user = userEvent.setup() + const onToggle = vi.fn().mockResolvedValue(undefined) + await render() + + await user.click(screen.getByRole('button', { name: /micronix/i })) + + expect(onToggle).toHaveBeenCalledWith(3, 'micronix_tube', true) + }) + + it('shows the error and keeps the saved state when turning a container type on fails', async () => { + const user = userEvent.setup() + const onToggle = vi.fn().mockRejectedValue(apiError(500, 'Database is locked')) + await render( + <> + + + + ) + + const micronix = screen.getByRole('button', { name: /micronix/i }) + await user.click(micronix) + + expect(await screen.findByRole('alert')).toHaveTextContent('Could not add Micronix Tube: Database is locked') + await waitFor(() => expect(micronix).toBeEnabled()) + expect(micronix).toHaveAttribute('aria-pressed', 'false') + }) + + it('shows the error and keeps the saved state when turning a container type off fails', async () => { + const user = userEvent.setup() + const onToggle = vi.fn().mockRejectedValue(new Error('Network Error')) + await render( + <> + + + + ) + + const cryovial = screen.getByRole('button', { name: /cryovial/i }) + await user.click(cryovial) + + expect(onToggle).toHaveBeenCalledWith(3, 'cryovial_tube', false) + expect(await screen.findByRole('alert')).toHaveTextContent('Could not remove Cryovial Tube: Network Error') + await waitFor(() => expect(cryovial).toBeEnabled()) + expect(cryovial).toHaveAttribute('aria-pressed', 'true') + }) +}) diff --git a/packages/web/src/components/__tests__/ReferenceDataForm.test.tsx b/packages/web/src/components/__tests__/ReferenceDataForm.test.tsx index 5ad148d0..ce74b96e 100644 --- a/packages/web/src/components/__tests__/ReferenceDataForm.test.tsx +++ b/packages/web/src/components/__tests__/ReferenceDataForm.test.tsx @@ -111,7 +111,7 @@ describe('ReferenceDataForm', () => { expect(valueInput.value).toBe('50') }) - const saveButton = screen.getByText('Create') + const saveButton = screen.getByRole('button', { name: 'Add' }) expect(saveButton).not.toBeDisabled() await user.click(saveButton) @@ -199,7 +199,7 @@ describe('ReferenceDataForm', () => { const nameInput = screen.getByLabelText(/^Name/) await user.clear(nameInput) await user.type(nameInput, 'Test') - const saveButton = screen.getByText('Create') + const saveButton = screen.getByRole('button', { name: 'Add' }) await user.click(saveButton) await waitFor(() => { @@ -225,7 +225,7 @@ describe('ReferenceDataForm', () => { ) await user.type(screen.getByLabelText(/^Name/), 'Test') - const saveButton = screen.getByText('Create') + const saveButton = screen.getByRole('button', { name: 'Add' }) await user.click(saveButton) // Button should be disabled during save @@ -233,7 +233,34 @@ describe('ReferenceDataForm', () => { expect(saveButton).toBeDisabled() }) }) -}) + describe('submit button label', () => { + const renderForm = (item: TestItem | Record | null) => + render( + + item={item as TestItem | null} + fields={fields as any} + onSave={vi.fn().mockResolvedValue(undefined)} + onCancel={vi.fn()} + title="Item" + /> + ) + + it('says Add for the empty item the Add New button opens', async () => { + await renderForm({}) + expect(screen.getByRole('button', { name: 'Add' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Update' })).not.toBeInTheDocument() + }) + it('says Add when there is no item', async () => { + await renderForm(null) + expect(screen.getByRole('button', { name: 'Add' })).toBeInTheDocument() + }) + it('says Update when editing a saved item', async () => { + await renderForm({ id: 1, name: 'Existing Item' }) + expect(screen.getByRole('button', { name: 'Update' })).toBeInTheDocument() + expect(screen.queryByRole('button', { name: 'Add' })).not.toBeInTheDocument() + }) + }) +}) diff --git a/packages/web/src/pages/AdminUsers.tsx b/packages/web/src/pages/AdminUsers.tsx index f019390b..fe70a3ce 100644 --- a/packages/web/src/pages/AdminUsers.tsx +++ b/packages/web/src/pages/AdminUsers.tsx @@ -25,6 +25,11 @@ interface EditUserData { export default function AdminUsers() { const queryClient = useQueryClient() const [mutationError, setMutationError] = useState(null) + /** Error from the open modal's request, shown inside that modal. */ + const [modalError, setModalError] = useState(null) + const [modalPending, setModalPending] = useState(false) + // State updates land after the click handler returns, so a ref blocks a second click in the same tick. + const modalPendingRef = useRef(false) const [searchQuery, setSearchQuery] = useState('') const [showDeleted, setShowDeleted] = useState(false) const [showCreateModal, setShowCreateModal] = useState(false) @@ -70,62 +75,60 @@ export default function AdminUsers() { ) }, [users, searchQuery]) - const handleCreate = async () => { + /** + * Runs a modal's request once: ignores clicks while one is pending and shows a failure inside the modal. + */ + const runModalRequest = async (fallbackMessage: string, request: () => Promise) => { + if (modalPendingRef.current) return + modalPendingRef.current = true + setModalPending(true) + setModalError(null) try { - if (!createForm.email || !createForm.name || !createForm.password) { - setMutationError('All fields are required') - return - } + await request() + } catch (err: unknown) { + setModalError(apiErrorMessage(err, fallbackMessage)) + } finally { + modalPendingRef.current = false + setModalPending(false) + } + } - if (createForm.password.length < 8) { - setMutationError('Password must be at least 8 characters') - return - } + const handleCreate = async () => { + if (!createForm.email || !createForm.name || !createForm.password) { + setModalError('All fields are required') + return + } + if (createForm.password.length < 8) { + setModalError('Password must be at least 8 characters') + return + } + + await runModalRequest('Failed to create user', async () => { await adminApi.createUser(createForm) - setShowCreateModal(false) - setCreateForm({ email: '', name: '', password: '', role: 'member' }) - setMutationError(null) + closeCreateModal() refreshUsers() - } catch (err: unknown) { - const message = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string } } }).response?.data?.error - : null - setMutationError(message || 'Failed to create user') - } + }) } const handleEdit = async () => { if (!selectedUser) return - try { + await runModalRequest('Failed to update user', async () => { await adminApi.updateUser(selectedUser.id, editForm) - setShowEditModal(false) - setSelectedUser(null) - setEditForm({}) + closeEditModal() refreshUsers() - } catch (err: unknown) { - const message = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string } } }).response?.data?.error - : null - setMutationError(message || 'Failed to update user') - } + }) } const handleDelete = async () => { if (!selectedUser) return - try { + await runModalRequest('Failed to delete user', async () => { await adminApi.deleteUser(selectedUser.id) - setShowDeleteModal(false) - setSelectedUser(null) + closeDeleteModal() refreshUsers() - } catch (err: unknown) { - const res = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string; details?: string } } }).response?.data - : null - setMutationError(res?.error || res?.details || 'Failed to delete user') - } + }) } const handleApprove = async (user: User) => { @@ -155,27 +158,20 @@ export default function AdminUsers() { const handleResetPassword = async () => { if (!selectedUser) return - try { - if (!passwordForm.password || passwordForm.password !== passwordForm.confirmPassword) { - setMutationError('Passwords do not match') - return - } + if (!passwordForm.password || passwordForm.password !== passwordForm.confirmPassword) { + setModalError('Passwords do not match') + return + } - if (passwordForm.password.length < 8) { - setMutationError('Password must be at least 8 characters') - return - } + if (passwordForm.password.length < 8) { + setModalError('Password must be at least 8 characters') + return + } + await runModalRequest('Failed to reset password', async () => { await adminApi.resetPassword(selectedUser.id, passwordForm.password) - setShowPasswordModal(false) - setSelectedUser(null) - setPasswordForm({ password: '', confirmPassword: '' }) - } catch (err: unknown) { - const message = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string } } }).response?.data?.error - : null - setMutationError(message || 'Failed to reset password') - } + closePasswordModal() + }) } const loadSessions = async (userId: number) => { @@ -184,30 +180,61 @@ export default function AdminUsers() { const response = await adminApi.getUserSessions(userId) setSessions(response.sessions) } catch (err: unknown) { - const message = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string } } }).response?.data?.error - : null - setMutationError(message || 'Failed to load sessions') + setModalError(apiErrorMessage(err, 'Failed to load sessions')) } finally { setSessionsLoading(false) } } const handleRevokeSession = async (sessionId: string) => { - try { + await runModalRequest('Failed to revoke session', async () => { await adminApi.revokeSession(sessionId) if (selectedUser) { await loadSessions(selectedUser.id) } - } catch (err: unknown) { - const message = err && typeof err === 'object' && 'response' in err - ? (err as { response?: { data?: { error?: string } } }).response?.data?.error - : null - setMutationError(message || 'Failed to revoke session') - } + }) + } + + const openCreateModal = () => { + setModalError(null) + setShowCreateModal(true) + } + + const closeCreateModal = () => { + setShowCreateModal(false) + setCreateForm({ email: '', name: '', password: '', role: 'member' }) + setModalError(null) + } + + const closeEditModal = () => { + setShowEditModal(false) + setSelectedUser(null) + setEditForm({}) + setModalError(null) + } + + const closeDeleteModal = () => { + setShowDeleteModal(false) + setSelectedUser(null) + setModalError(null) + } + + const closePasswordModal = () => { + setShowPasswordModal(false) + setSelectedUser(null) + setPasswordForm({ password: '', confirmPassword: '' }) + setModalError(null) + } + + const closeSessionsModal = () => { + setShowSessionsModal(false) + setSelectedUser(null) + setSessions([]) + setModalError(null) } const openEditModal = (user: User) => { + setModalError(null) setSelectedUser(user) setEditForm({ name: user.name, @@ -218,17 +245,20 @@ export default function AdminUsers() { } const openDeleteModal = (user: User) => { + setModalError(null) setSelectedUser(user) setShowDeleteModal(true) } const openPasswordModal = (user: User) => { + setModalError(null) setSelectedUser(user) setPasswordForm({ password: '', confirmPassword: '' }) setShowPasswordModal(true) } const openSessionsModal = async (user: User) => { + setModalError(null) setSelectedUser(user) setShowSessionsModal(true) await loadSessions(user.id) @@ -280,7 +310,7 @@ export default function AdminUsers() {

Manage users, roles, and permissions

@@ -831,11 +853,7 @@ export default function AdminUsers() { )}
) } + +/** The API's `error` (or `details`) message from a failed request, else the fallback. */ +function apiErrorMessage(err: unknown, fallback: string): string { + if (err && typeof err === 'object' && 'response' in err) { + const data = (err as { response?: { data?: { error?: string; details?: string } } }).response?.data + if (data?.error) return data.error + if (data?.details) return data.details + } + return fallback +} + +function ModalError({ message }: { message: string | null }) { + if (!message) return null + return ( +
+ {message} +
+ ) +} diff --git a/packages/web/src/pages/ReferenceData.tsx b/packages/web/src/pages/ReferenceData.tsx index b092fd07..326b1379 100644 --- a/packages/web/src/pages/ReferenceData.tsx +++ b/packages/web/src/pages/ReferenceData.tsx @@ -118,12 +118,16 @@ function ReferenceDataContent({ return } - if (isAdding) { - await specimenTypesApi.addContainerType(specimenTypeId, containerType) - } else { - await specimenTypesApi.removeContainerType(specimenTypeId, containerType) + try { + if (isAdding) { + await specimenTypesApi.addContainerType(specimenTypeId, containerType) + } else { + await specimenTypesApi.removeContainerType(specimenTypeId, containerType) + } + } finally { + // Refetch on failure too, so the toggle shows what the server saved. ContainerTypeToggle shows the error. + void invalidateAfterReferenceDataWrite(queryClient) } - void invalidateAfterReferenceDataWrite(queryClient) } useEffect(() => { diff --git a/packages/web/src/pages/__tests__/AdminUsers.test.tsx b/packages/web/src/pages/__tests__/AdminUsers.test.tsx index 3a0c0b2c..aac3db85 100644 --- a/packages/web/src/pages/__tests__/AdminUsers.test.tsx +++ b/packages/web/src/pages/__tests__/AdminUsers.test.tsx @@ -1,5 +1,5 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import AdminUsers from '../AdminUsers' import { adminApi } from '../../lib/api/admin' import userEvent from '@testing-library/user-event' @@ -71,4 +71,183 @@ describe('AdminUsers', () => { await waitFor(() => expect(adminApi.revokeSession).toHaveBeenCalledWith('pub_abc')) expect(await screen.findByText(/no active sessions/i)).toBeInTheDocument() }) + + describe('modal mutations', () => { + const member = { id: 7, name: 'Member User', email: 'member@test.com', role: 'member' as const, approvedAt: '2026-01-01' } + + function apiError(status: number, error: string) { + return Object.assign(new Error(`Request failed with status code ${status}`), { + response: { status, data: { error } }, + }) + } + + /** A promise the test settles by hand, to hold a request pending. */ + function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((r) => { resolve = r }) + return { promise, resolve } + } + + async function openCreateDialog(user: ReturnType) { + await user.click(await screen.findByRole('button', { name: /add user/i })) + const dialog = await screen.findByRole('dialog') + await user.type(within(dialog).getByLabelText('Name'), 'New Person') + await user.type(within(dialog).getByLabelText('Email'), 'new@test.com') + await user.type(within(dialog).getByLabelText('Password'), 'longenough') + return dialog + } + + beforeEach(() => { + vi.mocked(adminApi.getUsers).mockResolvedValue({ users: [member] }) + }) + + it('shows a failed create inside the create dialog', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.createUser).mockRejectedValueOnce(apiError(409, 'Email already registered')) + await render() + + const dialog = await openCreateDialog(user) + await user.click(within(dialog).getByRole('button', { name: 'Create' })) + + expect(await within(dialog).findByRole('alert')).toHaveTextContent('Email already registered') + }) + + it('shows a create validation error inside the create dialog', async () => { + const user = userEvent.setup() + await render() + + await user.click(await screen.findByRole('button', { name: /add user/i })) + const dialog = await screen.findByRole('dialog') + await user.click(within(dialog).getByRole('button', { name: 'Create' })) + + expect(await within(dialog).findByRole('alert')).toHaveTextContent('All fields are required') + expect(adminApi.createUser).not.toHaveBeenCalled() + }) + + it('shows a failed edit inside the edit dialog', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.updateUser).mockRejectedValueOnce(apiError(409, 'Email already in use')) + await render() + + await user.click(await screen.findByRole('button', { name: /edit user/i })) + const dialog = await screen.findByRole('dialog') + await user.click(within(dialog).getByRole('button', { name: 'Save' })) + + expect(await within(dialog).findByRole('alert')).toHaveTextContent('Email already in use') + }) + + it('shows a failed delete inside the delete dialog', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.deleteUser).mockRejectedValueOnce(apiError(400, 'Cannot delete the last admin')) + await render() + + await user.click(await screen.findByRole('button', { name: /delete user/i })) + const dialog = await screen.findByRole('dialog') + await user.click(within(dialog).getByRole('button', { name: 'Delete' })) + + expect(await within(dialog).findByRole('alert')).toHaveTextContent('Cannot delete the last admin') + }) + + it('shows a failed password reset inside the reset dialog', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.resetPassword).mockRejectedValueOnce(apiError(500, 'Could not reset password')) + await render() + + await user.click(await screen.findByRole('button', { name: /reset password/i })) + const dialog = await screen.findByRole('dialog') + await user.type(within(dialog).getByLabelText('New Password'), 'longenough') + await user.type(within(dialog).getByLabelText('Confirm Password'), 'longenough') + await user.click(within(dialog).getByRole('button', { name: 'Reset Password' })) + + expect(await within(dialog).findByRole('alert')).toHaveTextContent('Could not reset password') + }) + + it('clears a dialog error when the dialog is reopened', async () => { + const user = userEvent.setup() + vi.mocked(adminApi.updateUser).mockRejectedValueOnce(apiError(409, 'Email already in use')) + await render() + + await user.click(await screen.findByRole('button', { name: /edit user/i })) + let dialog = await screen.findByRole('dialog') + await user.click(within(dialog).getByRole('button', { name: 'Save' })) + await within(dialog).findByRole('alert') + await user.click(within(dialog).getByRole('button', { name: 'Cancel' })) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + + await user.click(screen.getByRole('button', { name: /edit user/i })) + dialog = await screen.findByRole('dialog') + expect(within(dialog).queryByRole('alert')).not.toBeInTheDocument() + }) + + it('disables Create while the request is pending and sends one request on a double click', async () => { + const user = userEvent.setup() + const pending = deferred<{ user: typeof member }>() + vi.mocked(adminApi.createUser).mockReturnValueOnce(pending.promise) + await render() + + const dialog = await openCreateDialog(user) + const create = within(dialog).getByRole('button', { name: 'Create' }) + await user.dblClick(create) + + expect(create).toBeDisabled() + expect(adminApi.createUser).toHaveBeenCalledTimes(1) + pending.resolve({ user: { ...member, id: 8, name: 'New Person', email: 'new@test.com' } }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + expect(adminApi.createUser).toHaveBeenCalledTimes(1) + }) + + it('disables Save while the edit is pending and sends one request on a double click', async () => { + const user = userEvent.setup() + const pending = deferred<{ user: typeof member }>() + vi.mocked(adminApi.updateUser).mockReturnValueOnce(pending.promise) + await render() + + await user.click(await screen.findByRole('button', { name: /edit user/i })) + const dialog = await screen.findByRole('dialog') + const save = within(dialog).getByRole('button', { name: 'Save' }) + await user.dblClick(save) + + expect(save).toBeDisabled() + expect(adminApi.updateUser).toHaveBeenCalledTimes(1) + pending.resolve({ user: member }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + expect(adminApi.updateUser).toHaveBeenCalledTimes(1) + }) + + it('disables Delete while the request is pending and sends one request on a double click', async () => { + const user = userEvent.setup() + const pending = deferred<{ message: string }>() + vi.mocked(adminApi.deleteUser).mockReturnValueOnce(pending.promise) + await render() + + await user.click(await screen.findByRole('button', { name: /delete user/i })) + const dialog = await screen.findByRole('dialog') + const del = within(dialog).getByRole('button', { name: 'Delete' }) + await user.dblClick(del) + + expect(del).toBeDisabled() + expect(adminApi.deleteUser).toHaveBeenCalledTimes(1) + pending.resolve({ message: 'User deleted' }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + }) + + it('disables Reset Password while the request is pending and sends one request on a double click', async () => { + const user = userEvent.setup() + const pending = deferred<{ message: string }>() + vi.mocked(adminApi.resetPassword).mockReturnValueOnce(pending.promise) + await render() + + await user.click(await screen.findByRole('button', { name: /reset password/i })) + const dialog = await screen.findByRole('dialog') + await user.type(within(dialog).getByLabelText('New Password'), 'longenough') + await user.type(within(dialog).getByLabelText('Confirm Password'), 'longenough') + const reset = within(dialog).getByRole('button', { name: 'Reset Password' }) + await user.dblClick(reset) + + expect(reset).toBeDisabled() + expect(adminApi.resetPassword).toHaveBeenCalledTimes(1) + pending.resolve({ message: 'Password reset' }) + await waitFor(() => expect(screen.queryByRole('dialog')).not.toBeInTheDocument()) + }) + }) }) diff --git a/packages/web/src/pages/__tests__/ReferenceData.test.tsx b/packages/web/src/pages/__tests__/ReferenceData.test.tsx index bc555ea9..190e988e 100644 --- a/packages/web/src/pages/__tests__/ReferenceData.test.tsx +++ b/packages/web/src/pages/__tests__/ReferenceData.test.tsx @@ -6,6 +6,7 @@ import { getReferenceDataConfig } from '../../config/reference-data-config' import { specimenTypeKeys, tagKeys } from '../../hooks/useReferenceData' import { referenceDataPageKeys } from '../../hooks/useReferenceDataPage' import { exportWorkflowKeys } from '../../hooks/useExportWorkflow' +import { specimenTypesApi } from '../../lib/api/reference-data' import { createInvalidationTestClient, keysNotInvalidated, @@ -48,6 +49,16 @@ vi.mock('../../hooks/useReferenceDataPage', async () => { } }) +vi.mock('../../lib/api/reference-data', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('reference-data', { + specimenTypesApi: { + addContainerType: vi.fn().mockResolvedValue(undefined), + removeContainerType: vi.fn().mockResolvedValue(undefined), + }, + }) +}) + // Mock the config module vi.mock('../../config/reference-data-config', () => { const createMockConfig = (id: string, label: string, options: any = {}) => ({ @@ -260,9 +271,8 @@ describe('ReferenceData Page', () => { seedQueries(queryClient, otherPages) await render(, { queryClient, initialEntries: ['/reference-data?tab=specimen-types'] }) - await user.click(screen.getByText('Add New')) - // The add form labels its submit button Update, since the page passes it an empty item. - await user.click(await screen.findByRole('button', { name: /^(Create|Update)$/ })) + await user.click(screen.getByRole('button', { name: 'Add New' })) + await user.click(await screen.findByRole('button', { name: 'Add' })) await waitFor(() => expect(getReferenceDataConfig('specimen-types')?.create).toHaveBeenCalled()) await waitFor(() => expect(keysNotInvalidated(queryClient, otherPages)).toEqual([])) @@ -297,6 +307,37 @@ describe('ReferenceData Page', () => { }) }) + describe('Container type toggles', () => { + /** The toggle handler the page passes to the specimen types table. */ + function toggleHandler() { + const { calls } = vi.mocked(getReferenceDataConfig('specimen-types')!.getColumns).mock + return calls[calls.length - 1]?.[0]?.onToggleContainerType + } + + it('refetches the saved container types when a toggle fails', async () => { + vi.mocked(specimenTypesApi.addContainerType).mockRejectedValueOnce(new Error('Database is locked')) + mockUseReferenceDataTab.mockReturnValue({ + data: [{ id: 3, name: 'Plasma' }], + isPending: false, + isLoading: false, + isError: false, + isSuccess: true, + error: null, + refetch: vi.fn(), + } as unknown as ReturnType) + const queryClient = createInvalidationTestClient() + const containerTypes = await observeQuery(queryClient, referenceDataPageKeys.containerTypes(3)) + await render(, { queryClient, initialEntries: ['/reference-data?tab=specimen-types'] }) + + const onToggle = toggleHandler() + expect(onToggle).toBeDefined() + await expect(onToggle!(3, 'micronix_tube', true)).rejects.toThrow('Database is locked') + + await waitFor(() => expect(containerTypes.queryFn).toHaveBeenCalledTimes(2)) + containerTypes.unsubscribe() + }) + }) + describe('Locations Tab', () => { it('renders locations tab when clicked', async () => { const user = userEvent.setup() From e30e79a2445c7e671c7c40b0627241adfa38345e Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:15:52 -0800 Subject: [PATCH 35/54] test(api): give createTestSpecimen's default Study a unique title The title was `Study ${Date.now()}-${random 0-999}`, so two calls in the same millisecond collided one time in a thousand and failed with UNIQUE constraint failed: study.title (seen in dashboard-stats under ci:verify). Number them with a counter instead. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/__tests__/helpers/factories.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/api/src/__tests__/helpers/factories.ts b/packages/api/src/__tests__/helpers/factories.ts index d8614505..fc1451cb 100644 --- a/packages/api/src/__tests__/helpers/factories.ts +++ b/packages/api/src/__tests__/helpers/factories.ts @@ -385,6 +385,9 @@ export async function createTestStudySubject( return result } +// Numbers the Studies createTestSpecimen makes, so two made in the same millisecond can't share a title. +let defaultSpecimenStudyCount = 0 + /** * Create a test specimen (for testing specimen type "in use") */ @@ -398,9 +401,10 @@ export async function createTestSpecimen( // Default to study subject if neither provided if (!studySubjectId && !controlBatchId) { + defaultSpecimenStudyCount += 1 const testStudy = await createTestStudy(db, { - title: `Study ${Date.now()}-${Math.floor(Math.random() * 1000)}`, - shortCode: `ST${Date.now()}-${Math.floor(Math.random() * 1000)}` + title: `Default specimen study ${defaultSpecimenStudyCount}`, + shortCode: `DSS${defaultSpecimenStudyCount}`, }); const testSubject = await createTestStudySubject(db, { studyId: testStudy.id, From 06de8730ec17fc30989ea554a74e35d96340a164 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:17:09 -0800 Subject: [PATCH 36/54] fix(controls): unique names, zero quantities and spreadsheet lines for control batch CSVs (#136) - Add batches from CSV no longer asks for a suggested name per batch before submit, so two files with the same density sent the same name and the create-many request failed on the unique name. The create-many schema now lets a batch omit its name, and the server names each batch inside the one transaction (`D100 2024-06-01`, then `D100 2024-06-01 (1)`). - A CSV quantity of 0 was replaced with 1 (`|| 1`); it is now kept, since the shared container quantity bounds allow 0. CSV quantity checks use containerQuantityBoundsError. - Each parsed CSV row keeps its spreadsheet line, so validation errors after blank or skipped rows name the line the user sees. - Document batch naming, zero quantities and row numbers in the Blood Controls guide. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/batch-with-specimens.test.ts | 27 ++++++ .../api/src/lib/controls/batch-schemas.ts | 14 ++- .../src/lib/controls/batch-with-specimens.ts | 6 +- .../api/src/routes/__tests__/controls.test.ts | 42 ++++++++- .../docs/guides/features/blood-controls.md | 4 + .../web/src/components/wizards/ReviewStep.tsx | 13 ++- .../wizards/__tests__/ReviewStep.test.tsx | 92 ++++++++++++++++++- .../lib/__tests__/control-batch-csv.test.ts | 45 +++++++++ packages/web/src/lib/api/controls.ts | 7 +- packages/web/src/lib/control-batch-csv.ts | 16 +++- 10 files changed, 247 insertions(+), 19 deletions(-) diff --git a/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts b/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts index dcb38332..8f385c8f 100644 --- a/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts +++ b/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts @@ -432,6 +432,33 @@ describe('batch-with-specimens', () => { expect(await testDb.select().from(storageContainer).all()).toHaveLength(0) }) + it('names each unnamed batch uniquely, though two batches share a definition and date', async () => { + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await setContainerDefaults(testDb, { + micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + }) + await testDb.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const def100 = await createTestControlDefinition(testDb, { name: 'D100' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'DNA' }) + await testDb.insert(specimenTypeContainerType).values({ specimenTypeId: specimenType.id, containerType: 'micronix_tube', created: utcNow() }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: location.id }) + const tube = (barcode: string, position: string) => ({ + containerType: 'micronix_tube' as const, + barcode, + collection: { type: 'micronix_plate' as const, id: plate.id, position }, + }) + + const results = await createBatchesWithSpecimens(testDb, [ + { batch: { controlDefinitionId: def100.id, productionDate: '2024-06-01' }, specimens: [{ specimenTypeName: 'DNA', containers: [tube('MT1', 'A01')] }] }, + { batch: { controlDefinitionId: def100.id, productionDate: '2024-06-01' }, specimens: [{ specimenTypeName: 'DNA', containers: [tube('MT2', 'A02')] }] }, + ]) + + expect(results.map((r) => r.batch.name)).toEqual(['D100 2024-06-01', 'D100 2024-06-01 (1)']) + expect(await testDb.select().from(controlBatch).all()).toHaveLength(2) + }) + it('reports a UNIQUE violation as a conflict', async () => { const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) await setContainerDefaults(testDb, { diff --git a/packages/api/src/lib/controls/batch-schemas.ts b/packages/api/src/lib/controls/batch-schemas.ts index e21f3e81..6d143d43 100644 --- a/packages/api/src/lib/controls/batch-schemas.ts +++ b/packages/api/src/lib/controls/batch-schemas.ts @@ -73,9 +73,19 @@ export const createBatchWithSpecimensSchema = z.object({ specimens: z.array(batchSpecimenInputSchema).min(1), }) -/** Several batches (e.g. one per density from a composition CSV), created all-or-nothing. */ +/** + * Several batches (e.g. one per density from a composition CSV), created all-or-nothing. + * A batch may omit its name; the server then names it inside the same transaction, so two + * batches for one definition and production date get different names. + */ export const createBatchesWithSpecimensSchema = z.object({ - batches: z.array(createBatchWithSpecimensSchema).min(1), + batches: z + .array( + createBatchWithSpecimensSchema.extend({ + batch: createBatchWithSpecimensSchema.shape.batch.extend({ name: z.string().min(1).optional() }), + }), + ) + .min(1), }) export const addSpecimensToBatchSchema = z.object({ diff --git a/packages/api/src/lib/controls/batch-with-specimens.ts b/packages/api/src/lib/controls/batch-with-specimens.ts index 256c233c..85561780 100644 --- a/packages/api/src/lib/controls/batch-with-specimens.ts +++ b/packages/api/src/lib/controls/batch-with-specimens.ts @@ -32,7 +32,8 @@ import { insertedRow } from '../inserted-row' export interface CreateBatchWithSpecimensRequest { batch: { controlDefinitionId: number - name: string + /** Omit to have the server generate a unique name from the definition and production date. */ + name?: string productionDate?: string properties?: Record } @@ -386,7 +387,8 @@ export async function createBatchesWithSpecimens( const conflict = error instanceof Error ? sqliteConstraintConflict(error) : null const reason = conflict ?? (error instanceof Error ? error.message : String(error)) const sentence = /[.!?]$/.test(reason) ? reason : `${reason}.` - const message = `Control batch ${index + 1} ('${data.batch.name}'): ${sentence} No control batches were created.` + const label = data.batch.name ? ` ('${data.batch.name}')` : '' + const message = `Control batch ${index + 1}${label}: ${sentence} No control batches were created.` if (conflict || error instanceof ConflictError) throw new ConflictError(message) throw new ValidationError(message) } diff --git a/packages/api/src/routes/__tests__/controls.test.ts b/packages/api/src/routes/__tests__/controls.test.ts index 84542c00..2af97da4 100644 --- a/packages/api/src/routes/__tests__/controls.test.ts +++ b/packages/api/src/routes/__tests__/controls.test.ts @@ -6,12 +6,17 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createControlsRoutes } from '../controls' -import { controlBatch, controlDefinition, specimen } from '../../db/schema' +import { containerTypeUnit, controlBatch, controlDefinition, specimen, specimenTypeContainerType } from '../../db/schema' +import { setContainerDefaults } from '../../lib/settings' +import { utcNow } from '../../lib/datetime' import { createTestControlBatch, createTestControlDefinition, + createTestLocation, + createTestMicronixPlate, createTestSpecimen, createTestSpecimenType, + createTestStorageType, createTestStrain, createTestUnit, } from '../../__tests__/helpers/factories' @@ -95,6 +100,41 @@ describe('Controls API', () => { }) }) + describe(`POST ${BASE}/batches/create-many-with-specimens`, () => { + it('gives two unnamed batches with the same definition and date different names', async () => { + const unit = await createTestUnit(ctx.db, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await setContainerDefaults(ctx.db, { + micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + }) + await ctx.db.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const def = await createTestControlDefinition(ctx.db, { name: 'D100' }) + const specimenType = await createTestSpecimenType(ctx.db, { name: 'DNA' }) + await ctx.db.insert(specimenTypeContainerType).values({ specimenTypeId: specimenType.id, containerType: 'micronix_tube', created: utcNow() }) + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + const location = await createTestLocation(ctx.db, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'Plate1', locationId: location.id }) + const tube = (barcode: string, position: string) => ({ + containerType: 'micronix_tube', + barcode, + collection: { type: 'micronix_plate', id: plate.id, position }, + }) + + const res = await ctx.request(`${BASE}/batches/create-many-with-specimens`, { + method: 'POST', + json: { + batches: [ + { batch: { controlDefinitionId: def.id, productionDate: '2024-06-01' }, specimens: [{ specimenTypeName: 'DNA', containers: [tube('MT1', 'A01')] }] }, + { batch: { controlDefinitionId: def.id, productionDate: '2024-06-01' }, specimens: [{ specimenTypeName: 'DNA', containers: [tube('MT2', 'A02')] }] }, + ], + }, + }) + + expect(res.status).toBe(201) + const data = (await res.json()) as { batches: Array<{ batch: { name: string } }> } + expect(data.batches.map((b) => b.batch.name)).toEqual(['D100 2024-06-01', 'D100 2024-06-01 (1)']) + }) + }) + describe(`GET ${BASE}/batches`, () => { it('returns 200 and batches list when authenticated', async () => { const res = await ctx.request(`${BASE}/batches`, { diff --git a/packages/docs/src/content/docs/guides/features/blood-controls.md b/packages/docs/src/content/docs/guides/features/blood-controls.md index e6315572..933aa530 100644 --- a/packages/docs/src/content/docs/guides/features/blood-controls.md +++ b/packages/docs/src/content/docs/guides/features/blood-controls.md @@ -55,6 +55,10 @@ Optional **density** column: rows are grouped by density and one batch is create Review previews file, collection, density, definition, and specimen count. Submit uses your chosen definitions. It does not create definitions automatically. +Each batch is named from its definition and the production date, for example `D100 2024-06-01`. When two files have the same density, or a batch already has that name, the next batch gets a number: `D100 2024-06-01 (1)`. All the batches are created together, or none are. + +A `quantity` of `0` is kept as 0. A blank `quantity` uses 1. Errors name the row as your spreadsheet numbers it: the header is row 1, and blank rows count. + ### Collection resolution As you type a collection name, matches appear with location. Selecting one reuses it. For a new name, **Create new collection** plus a location creates it on submit. This applies to both flows so you do not duplicate collections of the same name and type. diff --git a/packages/web/src/components/wizards/ReviewStep.tsx b/packages/web/src/components/wizards/ReviewStep.tsx index 08ac3ad2..128d6890 100644 --- a/packages/web/src/components/wizards/ReviewStep.tsx +++ b/packages/web/src/components/wizards/ReviewStep.tsx @@ -6,7 +6,7 @@ import { } from '../../lib/control-batch-payload' import { settingsApi } from '../../lib/api/settings'; import { getQueryErrorMessage } from '../../ui' -import type { ControlDefinition, CreateBatchWithSpecimensRequest } from '../../lib/api/controls'; +import type { ControlDefinition, CreateManyBatchesItem } from '../../lib/api/controls'; import { normalizePosition, groupRowsByDensity } from '../../lib/control-batch-csv' import type { BatchInfo, SpecimenTypeConfig, CSVFileData, CompositionStrains } from '../../pages/ControlBatchWizard' import { localToday } from '../../lib/date-utils' @@ -149,14 +149,12 @@ export default function ReviewStep({ setSubmitting(true) try { - const batchPayloads: CreateBatchWithSpecimensRequest[] = [] + const batchPayloads: CreateManyBatchesItem[] = [] for (const batchRow of multiBatchRows) { const definitionId = batchRow.candidates.length === 1 ? batchRow.candidates[0]!.id : batchDefinitionSelections[batchRow.rowKey] if (definitionId == null) continue // eslint-disable-line @typescript-eslint/no-unnecessary-condition - const nameRes = await controlsApi.suggestBatchName(definitionId, productionDate) - const batchName = nameRes.name const specimensMap = new Map() const file = batchRow.file for (const row of batchRow.rows) { @@ -199,9 +197,10 @@ export default function ReviewStep({ containers, })) batchPayloads.push({ + // No name: the server names each batch inside the one transaction, so two files with + // the same density (same definition and date) get different names instead of colliding. batch: { controlDefinitionId: definitionId, - name: batchName, productionDate, }, specimens: specimens.map(({ specimenTypeName, containers }) => ({ @@ -297,7 +296,7 @@ export default function ReviewStep({ collectionType: file.collectionType, sublabel: row.sublabel, sheetName: (row.sheet_name as string | undefined) || file.sheetName, - quantity: row.quantity || 1, + quantity: row.quantity ?? 1, unitSymbol: row.unit_symbol || fallbackUnitForContainerType(file.containerType!), } : { @@ -308,7 +307,7 @@ export default function ReviewStep({ collectionType: file.collectionType, barcode: row.barcode, position: row.position ? normalizePosition(row.position) : undefined, - quantity: row.quantity || 1, + quantity: row.quantity ?? 1, unitSymbol: row.unit_symbol || fallbackUnitForContainerType(file.containerType!), } diff --git a/packages/web/src/components/wizards/__tests__/ReviewStep.test.tsx b/packages/web/src/components/wizards/__tests__/ReviewStep.test.tsx index 6b2db46f..5ad5e0dd 100644 --- a/packages/web/src/components/wizards/__tests__/ReviewStep.test.tsx +++ b/packages/web/src/components/wizards/__tests__/ReviewStep.test.tsx @@ -166,7 +166,6 @@ describe('ReviewStep multi-batch CSV', () => { // Both densities go in one all-or-nothing request. expect(mockCreateBatchesWithSpecimens).toHaveBeenCalledTimes(1) }) - expect(mockSuggestBatchName).toHaveBeenCalled() const batches = mockCreateBatchesWithSpecimens.mock.calls[0][0].batches as Array<{ batch: { controlDefinitionId: number } }> expect(batches).toHaveLength(2) const definitionIds = batches.map((b) => b.batch.controlDefinitionId) @@ -475,3 +474,94 @@ describe('ReviewStep CSV unit fallback', () => { expect(container.unitSymbol).toBe('µL') }) }) + +describe('ReviewStep multi-batch CSV batch names', () => { + const compositionStrains: CompositionStrains = [{ id: 1, percentage: 100 }] + + beforeEach(() => { + vi.clearAllMocks() + mockSuggestBatchName.mockResolvedValue({ name: 'Def 5000 2024-06-01' }) + mockCreateBatchesWithSpecimens.mockResolvedValue({ batches: [{ batch: { id: 101 }, specimens: [], createdCollections: [] }] }) + }) + + it('leaves naming to the server, so two files with the same density do not send the same name', async () => { + const user = userEvent.setup() + const def5000: ControlDefinition = { + id: 10, + name: 'Def 5000', + controlType: 'blood', + created: '', + lastUpdated: '', + targetDensity: 5000, + } + await render( + {}} + onCancel={() => {}} + onSuccess={() => {}} + isAddMode={false} + /> + ) + + await user.click(screen.getByRole('button', { name: /Create Batch/i })) + + await waitFor(() => { + expect(mockCreateBatchesWithSpecimens).toHaveBeenCalledTimes(1) + }) + const batches = mockCreateBatchesWithSpecimens.mock.calls[0][0].batches as Array<{ batch: { name?: string; productionDate?: string } }> + expect(batches).toHaveLength(2) + for (const { batch } of batches) { + expect(batch).not.toHaveProperty('name') + expect(batch.productionDate).toBe('2024-06-01') + } + }) +}) + +describe('ReviewStep CSV zero quantity', () => { + beforeEach(() => { + vi.clearAllMocks() + mockCreateBatchWithSpecimens.mockResolvedValue({ batch: { id: 102 }, specimens: [], createdCollections: [] }) + }) + + it('sends a quantity of 0 from a CSV row as 0', async () => { + const user = userEvent.setup() + await render( + {}} + onCancel={() => {}} + onSuccess={() => {}} + isAddMode={false} + /> + ) + + await user.click(await screen.findByRole('button', { name: /Create Batch/i })) + + await waitFor(() => { + expect(mockCreateBatchWithSpecimens).toHaveBeenCalledTimes(1) + }) + const specimens = mockCreateBatchWithSpecimens.mock.calls[0][0].specimens as Array<{ containers: Array<{ quantity?: number }> }> + const quantities = specimens.flatMap((s) => s.containers.map((c) => c.quantity)) + expect(quantities).toEqual([0, 0]) + }) +}) diff --git a/packages/web/src/lib/__tests__/control-batch-csv.test.ts b/packages/web/src/lib/__tests__/control-batch-csv.test.ts index eecf8529..8b2b456d 100644 --- a/packages/web/src/lib/__tests__/control-batch-csv.test.ts +++ b/packages/web/src/lib/__tests__/control-batch-csv.test.ts @@ -144,6 +144,51 @@ describe('control-batch-csv', () => { }) }) + describe('spreadsheet line numbers in errors', () => { + const availableTypes = [{ name: 'Whole Blood' }] + + it('reports the spreadsheet line of a row after blank lines and a skipped row', () => { + const csv = [ + 'specimen_type_name,position,quantity', // line 1 + 'Whole Blood,A01,1', // line 2 + '', // line 3 + ',A02,1', // line 4: missing specimen type, skipped by the parser + ',,', // line 5: blank cells + 'Unknown Type,A03,1', // line 6 + ].join('\n') + const parsed = parseContainerCSV(csv, 'f.csv') + expect(parsed.errors).toEqual([expect.objectContaining({ row: 4, field: 'specimen_type_name' })]) + + const errors = validateCSVRows(parsed.rows, availableTypes, 'micronix_tube') + expect(errors).toEqual([expect.objectContaining({ row: 6, field: 'specimen_type_name' })]) + }) + + it('reports the spreadsheet line of a row in a file without blank lines', () => { + const csv = 'specimen_type_name,position\nWhole Blood,A01\nWhole Blood,\n' + const parsed = parseContainerCSV(csv, 'f.csv') + const errors = validateCSVRows(parsed.rows, availableTypes, 'micronix_tube') + expect(errors).toEqual([expect.objectContaining({ row: 3, field: 'position' })]) + }) + }) + + describe('validateCSVRows quantity', () => { + const availableTypes = [{ name: 'Whole Blood' }] + + it('accepts a quantity of 0, as the shared container quantity bounds do', () => { + const parsed = parseContainerCSV('specimen_type_name,quantity\nWhole Blood,0', 'f.csv') + expect(parsed.errors).toEqual([]) + expect(parsed.rows[0]?.quantity).toBe(0) + expect(validateCSVRows(parsed.rows, availableTypes)).toEqual([]) + }) + + it('rejects a negative quantity with the shared bounds message', () => { + const parsed = parseContainerCSV('specimen_type_name,quantity\nWhole Blood,-2', 'f.csv') + expect(validateCSVRows(parsed.rows, availableTypes)).toEqual([ + { row: 2, field: 'quantity', error: 'Total quantity must be 0 or more (got -2).' }, + ]) + }) + }) + describe('validateCSVRows', () => { const availableTypes = [{ name: 'Whole Blood' }, { name: 'Plasma' }] diff --git a/packages/web/src/lib/api/controls.ts b/packages/web/src/lib/api/controls.ts index 5ca103e9..e073e83a 100644 --- a/packages/web/src/lib/api/controls.ts +++ b/packages/web/src/lib/api/controls.ts @@ -136,6 +136,11 @@ export type CreateBatchWithSpecimensRequest = { }> } +/** One batch in a create-many request. Omit `batch.name` to have the server name it uniquely. */ +export type CreateManyBatchesItem = Omit & { + batch: Omit & { name?: string } +} + export type CreateBatchWithSpecimensResponse = { batch: ControlBatch specimens: Array<{ id: number; specimenTypeName: string; containerCount: number; containerIds: number[] }> @@ -166,7 +171,7 @@ export const controlsApi = { createBatchWithSpecimens: (data: CreateBatchWithSpecimensRequest) => api.post('/blood-controls/batches/create-with-specimens', data), /** Create several batches (one per density) all-or-nothing. */ - createBatchesWithSpecimens: (data: { batches: CreateBatchWithSpecimensRequest[] }) => + createBatchesWithSpecimens: (data: { batches: CreateManyBatchesItem[] }) => api.post<{ batches: CreateBatchWithSpecimensResponse[] }>('/blood-controls/batches/create-many-with-specimens', data), addSpecimensToBatch: (batchId: number, data: { specimens: Array<{ diff --git a/packages/web/src/lib/control-batch-csv.ts b/packages/web/src/lib/control-batch-csv.ts index dc5971e6..52a55fdd 100644 --- a/packages/web/src/lib/control-batch-csv.ts +++ b/packages/web/src/lib/control-batch-csv.ts @@ -1,4 +1,4 @@ -import { parseCsv } from '@sampledb/contract' +import { containerQuantityBoundsError, parseCsv } from '@sampledb/contract' import { buildCsv } from './csv' export interface CSVContainerRow { @@ -12,6 +12,8 @@ export interface CSVContainerRow { density?: number /** For paper (DBS): sheet name for this row. Multiple rows can share a sheet; sheets go into the file's collection (box/bag). */ sheet_name?: string + /** The row's line in the file as a spreadsheet numbers it (header is line 1, blank lines count). */ + line?: number } /** @@ -152,6 +154,7 @@ export function parseContainerCSV(csvText: string, filename: string): ParsedCSVF const parsedRow: CSVContainerRow = { specimen_type_name: specimenTypeName, + line: i + 1, } if (positionIdx >= 0 && row[positionIdx]) { @@ -224,7 +227,9 @@ export function validateCSVRows( const specimenTypeNames = new Set(availableSpecimenTypes.map(t => t.name)) rows.forEach((row, index) => { - const rowNum = index + 2 // +2 because row 1 is header, and we're 0-indexed + // Rows from parseContainerCSV carry their spreadsheet line; skipped blank or invalid rows + // make the index drift from it. Fall back to the index for rows built elsewhere. + const rowNum = row.line ?? index + 2 // Validate specimen type exists if (!specimenTypeNames.has(row.specimen_type_name)) { @@ -249,12 +254,13 @@ export function validateCSVRows( // For paper (DBS), each row must have sheet_name OR the file can have a single sheet name (validated in wizard). // We don't require sheet_name here so one-sheet-per-file (wizard field) still works. - // Validate quantity if provided - if (row.quantity !== undefined && (isNaN(row.quantity) || row.quantity < 0)) { + // Validate quantity if provided, with the bounds every Container write path shares (0 is allowed) + const quantityError = row.quantity === undefined ? null : containerQuantityBoundsError({ totalQuantity: row.quantity }) + if (quantityError) { errors.push({ row: rowNum, field: 'quantity', - error: 'Quantity must be a positive number', + error: quantityError, }) } }) From 1bab280da5924cb25efa2cf740cf6ef115745bcb Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:17:13 -0800 Subject: [PATCH 37/54] test(api): stop createTestSpecimen's default studies colliding The default study's title and short code were Date.now() plus a random number below 1000. The specimens pagination test creates 60 in a few milliseconds, so two often collided on study.short_code (2 of 5 runs failed locally). Use a module counter instead. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/__tests__/helpers/factories.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/api/src/__tests__/helpers/factories.ts b/packages/api/src/__tests__/helpers/factories.ts index d8614505..28e0ce2d 100644 --- a/packages/api/src/__tests__/helpers/factories.ts +++ b/packages/api/src/__tests__/helpers/factories.ts @@ -385,6 +385,8 @@ export async function createTestStudySubject( return result } +let defaultSpecimenStudyCount = 0 + /** * Create a test specimen (for testing specimen type "in use") */ @@ -398,9 +400,12 @@ export async function createTestSpecimen( // Default to study subject if neither provided if (!studySubjectId && !controlBatchId) { + // A counter, not Date.now() plus a random number: tests create dozens of these within one + // millisecond, and random suffixes then collide on study's unique title and short_code. + defaultSpecimenStudyCount += 1 const testStudy = await createTestStudy(db, { - title: `Study ${Date.now()}-${Math.floor(Math.random() * 1000)}`, - shortCode: `ST${Date.now()}-${Math.floor(Math.random() * 1000)}` + title: `Study ${Date.now()}-${defaultSpecimenStudyCount}`, + shortCode: `ST${Date.now()}-${defaultSpecimenStudyCount}` }); const testSubject = await createTestStudySubject(db, { studyId: testStudy.id, From 3bf931600a575d7367adc43410fe0fbfbb063f7d Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:17:55 -0800 Subject: [PATCH 38/54] fix(api): check control batch properties, Location collection flag and derivation dates (#133) - Control batches store `properties` as a JSON object; batch-create and batch-with-specimens no longer JSON.stringify it before the JSON column. Migration 012 unwraps rows stored as a JSON string holding an object and leaves every other row alone, so it is idempotent. - PUT /api/locations/:id refuses `canContainCollections: false` with a 409 naming how many Collections the Location holds. - A derivationDate must be YYYY-MM-DD, optionally followed by a time (`isDateOrTimestamp` in lib/date-filters.ts). POST derive, PATCH derivation, the derivation CSV (per-row and shared setting) and createDerivation itself reject anything else with a 400. Co-Authored-By: Claude Opus 5.5 --- packages/api/initial_schema.sql | 2 +- packages/api/src/db/__tests__/client.test.ts | 2 +- .../src/db/__tests__/schema-evolution.test.ts | 73 +++++++++++++++- ...2_control_batch_properties_single_json.sql | 14 +++ packages/api/src/db/schema-evolution.ts | 2 +- .../src/lib/__tests__/derivations-csv.test.ts | 26 ++++++ .../__tests__/batch-with-specimens.test.ts | 46 ++++++++++ packages/api/src/lib/controls/batch-create.ts | 2 +- .../src/lib/controls/batch-with-specimens.ts | 6 +- packages/api/src/lib/date-filters.ts | 13 +++ packages/api/src/lib/derivation-schemas.ts | 8 +- packages/api/src/lib/derivations-csv.ts | 5 ++ packages/api/src/lib/derivations.ts | 6 +- packages/api/src/lib/location-helpers.ts | 9 ++ .../api/src/routes/__tests__/controls.test.ts | 20 +++++ .../src/routes/__tests__/derivations.test.ts | 44 ++++++++++ .../src/routes/__tests__/locations.test.ts | 87 ++++++++++++++++++- packages/api/src/routes/derivations.ts | 4 +- packages/api/src/routes/imports.ts | 4 +- packages/api/src/routes/locations.ts | 13 ++- 20 files changed, 370 insertions(+), 16 deletions(-) create mode 100644 packages/api/src/db/migrations/012_control_batch_properties_single_json.sql diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index 8b5d547f..ea80d50a 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -297,7 +297,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (11);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (12);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/client.test.ts b/packages/api/src/db/__tests__/client.test.ts index 0fa9d243..3ae770fa 100644 --- a/packages/api/src/db/__tests__/client.test.ts +++ b/packages/api/src/db/__tests__/client.test.ts @@ -38,7 +38,7 @@ describe('Database Client', () => { // Tables migration 011 indexes db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER)`) db.exec(`CREATE TABLE storage_container_tag (storage_container_id INTEGER, tag_id INTEGER)`) - db.exec(`CREATE TABLE control_batch (id INTEGER PRIMARY KEY, control_definition_id INTEGER)`) + db.exec(`CREATE TABLE control_batch (id INTEGER PRIMARY KEY, control_definition_id INTEGER, properties TEXT)`) db.exec(`CREATE TABLE location (id INTEGER PRIMARY KEY, parent_id INTEGER, path TEXT)`) db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER)`) db.exec(`CREATE TABLE cryovial_box (id INTEGER PRIMARY KEY, location_id INTEGER)`) diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index 4290d5dd..f3d3da92 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -11,7 +11,7 @@ import { isEmptyOperationalDatabase, } from '../schema-evolution' import { openOperationalDatabase } from '../open' -import { SchemaMigrationError, runSqlMigrationStatements } from '../migration-runner' +import { SchemaMigrationError, listNumberedMigrations, runSqlMigration, runSqlMigrationStatements } from '../migration-runner' /** Tables that migrations 004+ alter, so older-version fixtures can migrate to current. */ function createTablesForLaterMigrations(sqlite: Database): void { @@ -60,7 +60,7 @@ function createTablesForLaterMigrations(sqlite: Database): void { ensureColumns(sqlite, 'specimen', ['control_batch_id INTEGER']) ensureColumns(sqlite, 'storage_container', ['specimen_id INTEGER']) ensureColumns(sqlite, 'storage_container_tag', ['storage_container_id INTEGER', 'tag_id INTEGER']) - ensureColumns(sqlite, 'control_batch', ['control_definition_id INTEGER']) + ensureColumns(sqlite, 'control_batch', ['control_definition_id INTEGER', 'properties TEXT']) ensureColumns(sqlite, 'location', ['parent_id INTEGER', 'path TEXT']) ensureColumns(sqlite, 'micronix_plate', ['location_id INTEGER']) ensureColumns(sqlite, 'cryovial_box', ['location_id INTEGER']) @@ -548,6 +548,75 @@ describe('schema evolution', () => { }) } + /** A database at version 11 with control batches, ready for migration 012. */ + function databaseBeforeMigration012(): Database { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (11)') + createTablesForLaterMigrations(sqlite) + const insert = sqlite.prepare('INSERT INTO control_batch (id, control_definition_id, properties) VALUES (?, 1, ?)') + const object = { legacy_study_id: 94, lead_person: 'Ann', nested: { strains: [1, 2] } } + insert.run(1, JSON.stringify(object)) + insert.run(2, JSON.stringify(JSON.stringify(object))) + insert.run(3, null) + insert.run(4, JSON.stringify(JSON.stringify({}))) + insert.run(5, '{}') + return sqlite + } + + function storedProperties(sqlite: Database): Array<[number, string | null, string | null]> { + const rows = sqlite + .prepare(`SELECT id, properties, json_type(properties) AS type FROM control_batch ORDER BY id`) + .all() as Array<{ id: number; properties: string | null; type: string | null }> + return rows.map((r) => [r.id, r.properties, r.type]) + } + + const MIGRATED_PROPERTIES: Array<[number, string | null, string | null]> = [ + [1, '{"legacy_study_id":94,"lead_person":"Ann","nested":{"strains":[1,2]}}', 'object'], + [2, '{"legacy_study_id":94,"lead_person":"Ann","nested":{"strains":[1,2]}}', 'object'], + [3, null, null], + [4, '{}', 'object'], + [5, '{}', 'object'], + ] + + it('migration 012 stores double-encoded control batch properties once and leaves single-encoded ones alone', () => { + const sqlite = databaseBeforeMigration012() + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(storedProperties(sqlite)).toEqual(MIGRATED_PROPERTIES) + sqlite.close() + }) + + it('migration 012 changes nothing when run again', () => { + const sqlite = databaseBeforeMigration012() + evolveOperationalSchema(sqlite) + const migration = listNumberedMigrations().find((m) => m.version === 12) + expect(migration).toBeDefined() + + runSqlMigration(sqlite, 12, migration!.path) + + expect(storedProperties(sqlite)).toEqual(MIGRATED_PROPERTIES) + sqlite.close() + }) + + it('migration 012 leaves a JSON string property that does not hold an object as it is', () => { + const sqlite = databaseBeforeMigration012() + sqlite.prepare('INSERT INTO control_batch (id, control_definition_id, properties) VALUES (6, 1, ?)').run(JSON.stringify('not json {')) + sqlite.prepare('INSERT INTO control_batch (id, control_definition_id, properties) VALUES (7, 1, ?)').run(JSON.stringify('[1,2]')) + + evolveOperationalSchema(sqlite) + + const rows = storedProperties(sqlite) + expect(rows.slice(0, 5)).toEqual(MIGRATED_PROPERTIES) + expect(rows.slice(5)).toEqual([ + [6, '"not json {"', 'text'], + [7, '"[1,2]"', 'text'], + ]) + sqlite.close() + }) + it('fail-hard: invalid migration leaves schema_version unchanged', () => { const sqlite = new Database(testDbPath) sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') diff --git a/packages/api/src/db/migrations/012_control_batch_properties_single_json.sql b/packages/api/src/db/migrations/012_control_batch_properties_single_json.sql new file mode 100644 index 00000000..3d98ce74 --- /dev/null +++ b/packages/api/src/db/migrations/012_control_batch_properties_single_json.sql @@ -0,0 +1,14 @@ +-- Migration 012: control batch properties are stored as a JSON object, not as a JSON string that +-- holds one (issue #133). batch-create.ts and batch-with-specimens.ts called JSON.stringify on +-- `properties` before handing it to a JSON column, so the stored text was `"{\"a\":1}"` and reads +-- returned a string. Unwrap each such value to the object text it holds. A row already holding an +-- object, a NULL, or a JSON string that does not hold an object is left as it is, so running this +-- again changes nothing. The CASE guards keep json_type() off text that is not valid JSON, which +-- it would otherwise reject with an error. +UPDATE control_batch +SET properties = json_extract(properties, '$') +WHERE typeof(properties) = 'text' + AND CASE WHEN json_valid(properties) THEN json_type(properties) END = 'text' + AND CASE + WHEN json_valid(json_extract(properties, '$')) THEN json_type(json_extract(properties, '$')) + END = 'object'; diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index 0cbaaf71..260162b0 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -7,7 +7,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 11 +export const CURRENT_SCHEMA_VERSION = 12 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/lib/__tests__/derivations-csv.test.ts b/packages/api/src/lib/__tests__/derivations-csv.test.ts index 97a18657..e79ce643 100644 --- a/packages/api/src/lib/__tests__/derivations-csv.test.ts +++ b/packages/api/src/lib/__tests__/derivations-csv.test.ts @@ -390,6 +390,32 @@ describe('derivations-csv', () => { expect(child?.position).toBe('B01') }) + it('rejects a derivation_date that is not a date, in validation and on import', async () => { + const { parent } = await seedParent() + const perRow = { ...settings, derivationDate: '' } + const csv = [`${header},derivation_date`, `${parent.id},PlateDef,B01,C1,not a date`, `${parent.id},PlateDef,B02,C2,2025-01-16`].join('\n') + + const result = await validateDerivationsCsv(testDb, csv, perRow) + expect(result.rows.map((r) => r.valid)).toEqual([false, true]) + expect(result.rows[0].error).toBe("derivation_date 'not a date' is not a date (use YYYY-MM-DD)") + + const imported = await importDerivationsFromCsv(testDb, csv, { settings: perRow }) + expect(imported.rows[0]?.success).toBe(false) + expect(imported.rows[0]?.error).toContain("derivation_date 'not a date' is not a date") + expect(await testDb.select().from(containerDerivation)).toHaveLength(0) + }) + + it('refuses a shared derivation date that is not a date and saves nothing', async () => { + const { parent } = await seedParent() + const csv = [header, `${parent.id},PlateDef,B01,C1`].join('\n') + + const imported = await importDerivationsFromCsv(testDb, csv, { settings: { ...settings, derivationDate: '2025-02-30' } }) + + expect(imported.rows[0]?.success).toBe(false) + expect(imported.rows[0]?.error).toContain("'2025-02-30' is not a date") + expect(await testDb.select().from(containerDerivation)).toHaveLength(0) + }) + /** A cryovial tube with `barcode`, on the parent's Specimen. */ async function addCryovial(seed: Awaited>, barcode: string) { const cryoBox = await createTestCryovialBox(testDb, { name: 'BoxDef', locationId: seed.loc.id }) diff --git a/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts b/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts index dcb38332..ede906ff 100644 --- a/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts +++ b/packages/api/src/lib/controls/__tests__/batch-with-specimens.test.ts @@ -127,6 +127,52 @@ describe('batch-with-specimens', () => { expect(result.specimens[0].containerIds).toHaveLength(1) }) + it('stores batch properties as a JSON object and returns them as one', async () => { + const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) + await setContainerDefaults(testDb, { + micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + cryovial_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + paper: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + static_well: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' }, + }) + await testDb.insert(containerTypeUnit).values({ containerType: 'micronix_tube', unitId: unit.id }) + const definition = await createTestControlDefinition(testDb, { name: 'Def1' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'DNA' }) + await testDb.insert(specimenTypeContainerType).values({ + specimenTypeId: specimenType.id, + containerType: 'micronix_tube', + created: utcNow(), + }) + const storageType = await createTestStorageType(testDb, { name: 'Freezer' }) + const location = await createTestLocation(testDb, { name: 'Loc', storageTypeId: String(storageType.id) }) + const plate = await createTestMicronixPlate(testDb, { name: 'Plate1', locationId: location.id }) + const properties = { lead_person: 'Ann', density: 100 } + + const result = await createBatchWithSpecimens(testDb, { + batch: { controlDefinitionId: definition.id, name: 'Batch1', properties }, + specimens: [ + { + specimenTypeName: 'DNA', + containers: [ + { + containerType: 'micronix_tube', + barcode: 'MT001', + collection: { type: 'micronix_plate', id: plate.id, position: 'A01' }, + }, + ], + }, + ], + }) + + expect(result.batch.properties).toEqual(properties) + const stored = await testDb.select().from(controlBatch).where(eq(controlBatch.id, result.batch.id)).get() + expect(stored?.properties).toEqual(properties) + const raw = sqlite + .prepare('SELECT json_type(properties) AS type FROM control_batch WHERE id = ?') + .get(result.batch.id) as { type: string } + expect(raw.type).toBe('object') + }) + it('rejects one new plate given two locations, though each container is registered on its own', async () => { const unit = await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' }) await setContainerDefaults(testDb, { micronix_tube: { totalQuantity: 1, remainingQuantity: 1, defaultUnitSymbol: 'uL' } }) diff --git a/packages/api/src/lib/controls/batch-create.ts b/packages/api/src/lib/controls/batch-create.ts index 6f7bae3b..5ee7900f 100644 --- a/packages/api/src/lib/controls/batch-create.ts +++ b/packages/api/src/lib/controls/batch-create.ts @@ -49,7 +49,7 @@ export async function createBloodControlBatch( controlDefinitionId: definitionId, name: batchName, productionDate: input.productionDate || null, - properties: input.properties ? JSON.stringify(input.properties) : null, + properties: input.properties ?? null, createdBy: userId, updatedBy: userId, }) diff --git a/packages/api/src/lib/controls/batch-with-specimens.ts b/packages/api/src/lib/controls/batch-with-specimens.ts index 256c233c..5b49167c 100644 --- a/packages/api/src/lib/controls/batch-with-specimens.ts +++ b/packages/api/src/lib/controls/batch-with-specimens.ts @@ -196,7 +196,7 @@ export async function createBatchWithSpecimens( controlDefinitionId: number name: string productionDate: string | null - properties: string | null + properties: Record | null created: string lastUpdated: string } @@ -234,7 +234,7 @@ export async function createBatchWithSpecimens( controlDefinitionId: data.batch.controlDefinitionId, name: batchName, productionDate: data.batch.productionDate || null, - properties: data.batch.properties ? JSON.stringify(data.batch.properties) : null, + properties: data.batch.properties ?? null, }) .returning() @@ -281,7 +281,7 @@ export async function createBatchWithSpecimens( controlDefinitionId: batchRecord.controlDefinitionId, name: batchRecord.name, productionDate: batchRecord.productionDate, - properties: (batchRecord.properties as string | null) ?? null, + properties: (batchRecord.properties as Record | null) ?? null, created: batchRecord.created, lastUpdated: batchRecord.lastUpdated, }, diff --git a/packages/api/src/lib/date-filters.ts b/packages/api/src/lib/date-filters.ts index 043a640f..4a01776a 100644 --- a/packages/api/src/lib/date-filters.ts +++ b/packages/api/src/lib/date-filters.ts @@ -5,15 +5,28 @@ */ import { ValidationError } from './error-handler' import { parseCollectionDate } from './validation' +import { parseStoredTimestamp } from './datetime' const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/ const TIMESTAMP = /^(\d{4}-\d{2}-\d{2})(?:[T ](\d{2}:\d{2}(?::\d{2})?))?$/ +const DATE_OR_TIMESTAMP = /^(\d{4}-\d{2}-\d{2})(?:[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})?)?$/ /** True when `value` is a real calendar date written YYYY-MM-DD. */ export function isIsoCalendarDate(value: string): boolean { return ISO_DATE.test(value) && parseCollectionDate(value) === value } +/** + * True when `value` is a real calendar date written YYYY-MM-DD, optionally followed by a time + * (`YYYY-MM-DD HH:MM[:SS[.fff]]` or the ISO `T` form, with an optional `Z` or `±HH:MM` zone). + * Rejects text `parseStoredTimestamp` alone would still read, such as `2024-02-30` (rolled over to + * March 1), `3/5/2024` or `1`. + */ +export function isDateOrTimestamp(value: string): boolean { + const match = DATE_OR_TIMESTAMP.exec(value) + return match != null && isIsoCalendarDate(match[1]) && parseStoredTimestamp(value) !== null +} + /** * A collection-date filter bound. Empty or omitted means no bound; anything other than a real * YYYY-MM-DD date throws a `ValidationError` naming `label`. diff --git a/packages/api/src/lib/derivation-schemas.ts b/packages/api/src/lib/derivation-schemas.ts index eeb86558..2dc10b5e 100644 --- a/packages/api/src/lib/derivation-schemas.ts +++ b/packages/api/src/lib/derivation-schemas.ts @@ -1,5 +1,11 @@ import { z } from 'zod' import { containerWriteInputSchema } from '@sampledb/contract' +import { isDateOrTimestamp } from './date-filters' + +/** A Derivation date: YYYY-MM-DD, optionally with a time (see `isDateOrTimestamp`). */ +export const derivationDateSchema = z + .string() + .refine(isDateOrTimestamp, { message: 'derivationDate must be a date written YYYY-MM-DD, optionally followed by a time' }) const legacyDerivationPlacementKeys = [ 'containerType', @@ -42,7 +48,7 @@ export const createDerivationRequestSchema = z unitSymbol: z.string().optional(), quantityUsed: z.number().optional(), reduceParentQuantity: z.boolean().optional().default(true), - derivationDate: z.string().optional(), + derivationDate: derivationDateSchema.optional(), protocol: z.string().optional(), notes: z.string().optional(), properties: z.record(z.string(), z.any()).optional(), diff --git a/packages/api/src/lib/derivations-csv.ts b/packages/api/src/lib/derivations-csv.ts index 374dae0d..5a02513f 100644 --- a/packages/api/src/lib/derivations-csv.ts +++ b/packages/api/src/lib/derivations-csv.ts @@ -25,6 +25,7 @@ import { } from '@sampledb/contract' import { createDerivation, type CreateDerivationInput } from './derivations' import { withWriteTransaction } from '../db/write-transaction' +import { isDateOrTimestamp } from './date-filters' import { resolveParentContainerId } from './derivations/parent-container-resolver' import { findTubesUsingBarcode } from './identifier-resolution' import { checkGridPositionOccupancy } from './container-occupancy' @@ -175,6 +176,10 @@ function rowValueError(row: DerivationCsvRow): string | null { if (flag != null && flag.trim() !== '' && !BOOLEAN_WORDS.has(flag.trim().toLowerCase())) { return `reduce_parent_quantity '${flag}' must be true or false` } + const date = row.derivation_date + if (date != null && date.trim() !== '' && !isDateOrTimestamp(date)) { + return `derivation_date '${date}' is not a date (use YYYY-MM-DD)` + } return null } diff --git a/packages/api/src/lib/derivations.ts b/packages/api/src/lib/derivations.ts index 0e1854f7..8cc63f23 100644 --- a/packages/api/src/lib/derivations.ts +++ b/packages/api/src/lib/derivations.ts @@ -14,7 +14,7 @@ import { and, eq, isNull, sql } from 'drizzle-orm' import { validateContainerTypeForSpecimenType } from '../lib/validation' import { getDefaultUnit } from './defaults' import { utcNow } from './datetime' -import { isIsoCalendarDate } from './date-filters' +import { isDateOrTimestamp, isIsoCalendarDate } from './date-filters' import { createContainerForSpecimen, pickContainerQuantity } from './container-creation' import { withWriteTransaction } from '../db/write-transaction' import { insertedRow } from './inserted-row' @@ -222,6 +222,10 @@ async function createDerivationInTx( throw new Error('Parent container not found') } + if (input.derivationDate !== undefined && !isDateOrTimestamp(input.derivationDate)) { + throw new ValidationError(`Derivation date '${input.derivationDate}' is not a date (use YYYY-MM-DD)`) + } + const quantityError = derivationQuantityError(parent, input) if (quantityError) { throw new ValidationError(quantityError) diff --git a/packages/api/src/lib/location-helpers.ts b/packages/api/src/lib/location-helpers.ts index 74d5c120..3ef2aded 100644 --- a/packages/api/src/lib/location-helpers.ts +++ b/packages/api/src/lib/location-helpers.ts @@ -341,3 +341,12 @@ export async function getLocationHierarchyStats(database: Database, sqliteDataba } } +/** How many Collections (micronix plates, cryovial boxes, boxes and bags) sit directly at a Location. */ +export async function countCollectionsAtLocation(database: Database, locationId: number): Promise { + const counts = await Promise.all( + [micronixPlate, cryovialBox, box, bag].map((table) => + database.select({ count: sql`COUNT(*)` }).from(table).where(eq(table.locationId, locationId)).get(), + ), + ) + return counts.reduce((total, row) => total + (row?.count ?? 0), 0) +} diff --git a/packages/api/src/routes/__tests__/controls.test.ts b/packages/api/src/routes/__tests__/controls.test.ts index 84542c00..5bb571c7 100644 --- a/packages/api/src/routes/__tests__/controls.test.ts +++ b/packages/api/src/routes/__tests__/controls.test.ts @@ -128,6 +128,26 @@ describe('Controls API', () => { expect(data.batch.controlDefinitionId).toBe(definition.id) }) + it('stores batch properties as a JSON object and reads them back as one', async () => { + const definition = await createTestControlDefinition(ctx.db, { name: 'Props Def', controlType: 'blood' }) + const properties = { lead_person: 'Ann', nested: { strains: [1, 2] } } + + const res = await ctx.request(`${BASE}/${definition.id}/batches`, { + method: 'POST', + json: { productionDate: '2024-01-01', properties }, + }) + + expect(res.status).toBe(201) + const created = (await res.json()) as { batch: { id: number; properties: unknown } } + expect(created.batch.properties).toEqual(properties) + const stored = ctx.sqlite + .prepare('SELECT json_type(properties) AS type FROM control_batch WHERE id = ?') + .get(created.batch.id) as { type: string } + expect(stored.type).toBe('object') + const readRes = await ctx.request(`${BASE}/batches/${created.batch.id}`, { method: 'GET' }) + expect(((await readRes.json()) as { batch: { properties: unknown } }).batch.properties).toEqual(properties) + }) + it('returns 404 when definition does not exist', async () => { const res = await ctx.request(`${BASE}/99999/batches`, { method: 'POST', diff --git a/packages/api/src/routes/__tests__/derivations.test.ts b/packages/api/src/routes/__tests__/derivations.test.ts index b3cc5c27..b0f65f2a 100644 --- a/packages/api/src/routes/__tests__/derivations.test.ts +++ b/packages/api/src/routes/__tests__/derivations.test.ts @@ -206,6 +206,28 @@ describe('Derivations API', () => { expect(await parentRemaining()).toBe(0) }) + it.each(['not a date', '2024-02-30', '2024-13-01', '3/5/2024', '1', ''])( + 'returns 400 for a derivationDate of %j and changes nothing', + async (derivationDate) => { + const res = await derive({ derivationDate }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { errorCode: string }).errorCode).toBe('VALIDATION_ERROR') + await expectNothingDerived() + }, + ) + + it.each(['2024-01-02', '2024-01-02 10:30:00', '2024-01-02T10:30:00.000Z'])( + 'stores a derivationDate of %j as sent', + async (derivationDate) => { + const res = await derive({ derivationDate }) + + expect(res.status).toBe(200) + const [row] = await ctx.db.select().from(containerDerivation) + expect(row?.derivationDate).toBe(derivationDate) + }, + ) + it('does not bound a quantity used that leaves the Parent container unreduced', async () => { const res = await derive({ quantityUsed: 100, reduceParentQuantity: false }) @@ -274,6 +296,28 @@ describe('Derivations API', () => { expect((await loadDerivation(d.id))?.protocol).toBe('P1') }) + it.each(['not a date', '2024-02-30', ''])( + 'returns 400 for a derivationDate of %j and leaves the Derivation unchanged', + async (derivationDate) => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { + method: 'PATCH', + json: { derivationDate, protocol: 'P2' }, + }) + expect(res.status).toBe(400) + const row = await loadDerivation(d.id) + expect(row?.derivationDate).toBe(d.derivationDate) + expect(row?.protocol).toBe('P1') + }, + ) + + it('lets an admin change a Derivation date to a valid date', async () => { + const d = await createDerivationRow() + const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'PATCH', json: { derivationDate: '2024-03-05' } }) + expect(res.status).toBe(200) + expect((await loadDerivation(d.id))?.derivationDate).toBe('2024-03-05') + }) + it('returns 403 to a member deleting a Derivation and keeps it', async () => { const d = await createDerivationRow() const res = await ctx.request(`${BASE}/derivations/${d.id}`, { method: 'DELETE', cookie: ctx.cookies.member }) diff --git a/packages/api/src/routes/__tests__/locations.test.ts b/packages/api/src/routes/__tests__/locations.test.ts index 7e91f98c..7d29922f 100644 --- a/packages/api/src/routes/__tests__/locations.test.ts +++ b/packages/api/src/routes/__tests__/locations.test.ts @@ -5,7 +5,14 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createLocationsRoutes } from '../locations' -import { createTestStorageType, createTestLocation } from '../../__tests__/helpers/factories' +import { + createTestBag, + createTestBox, + createTestCryovialBox, + createTestLocation, + createTestMicronixPlate, + createTestStorageType, +} from '../../__tests__/helpers/factories' import { location } from '../../db/schema' import { eq } from 'drizzle-orm' @@ -137,4 +144,82 @@ describe('Locations API', () => { expect(row.storageTypeId).toBeNull() }) }) + describe('PUT /api/locations/:id canContainCollections', () => { + async function collectionLocation() { + const storageType = await createTestStorageType(ctx.db, { name: 'Freezer' }) + return createTestLocation(ctx.db, { name: 'Shelf', storageTypeId: String(storageType.id), canContainCollections: true }) + } + + async function loadLocation(id: number) { + return ctx.db.select().from(location).where(eq(location.id, id)).get() + } + + it('returns 409 naming how many Collections the Location holds, and changes nothing', async () => { + const shelf = await collectionLocation() + await createTestMicronixPlate(ctx.db, { name: 'P1', locationId: shelf.id }) + await createTestMicronixPlate(ctx.db, { name: 'P2', locationId: shelf.id }) + await createTestCryovialBox(ctx.db, { name: 'C1', locationId: shelf.id }) + await createTestBox(ctx.db, { name: 'B1', locationId: shelf.id }) + await createTestBag(ctx.db, { name: 'G1', locationId: shelf.id }) + + const res = await ctx.request(`/api/locations/${shelf.id}`, { + method: 'PUT', + cookie: ctx.cookies.admin, + json: { canContainCollections: false, description: 'changed' }, + }) + + expect(res.status).toBe(409) + const body = (await res.json()) as { error: string; errorCode: string } + expect(body.errorCode).toBe('CONFLICT') + expect(body.error).toBe( + 'Cannot turn off "can contain collections": this location holds 5 collections. Move them to another location first.', + ) + const row = await loadLocation(shelf.id) + expect(row?.canContainCollections).toBe(true) + expect(row?.description).toBe(shelf.description) + }) + + it('says "1 collection" when the Location holds one', async () => { + const shelf = await collectionLocation() + await createTestBag(ctx.db, { name: 'G1', locationId: shelf.id }) + + const res = await ctx.request(`/api/locations/${shelf.id}`, { + method: 'PUT', + cookie: ctx.cookies.admin, + json: { canContainCollections: false }, + }) + + expect(res.status).toBe(409) + expect(((await res.json()) as { error: string }).error).toContain('holds 1 collection.') + }) + + it('turns off canContainCollections on a Location that holds no Collections', async () => { + const shelf = await collectionLocation() + const other = await createTestLocation(ctx.db, { name: 'Other', storageTypeId: shelf.storageTypeId, canContainCollections: true }) + await createTestMicronixPlate(ctx.db, { name: 'P1', locationId: other.id }) + + const res = await ctx.request(`/api/locations/${shelf.id}`, { + method: 'PUT', + cookie: ctx.cookies.admin, + json: { canContainCollections: false }, + }) + + expect(res.status).toBe(200) + expect((await loadLocation(shelf.id))?.canContainCollections).toBe(false) + }) + + it('accepts canContainCollections: true on a Location that holds Collections', async () => { + const shelf = await collectionLocation() + await createTestMicronixPlate(ctx.db, { name: 'P1', locationId: shelf.id }) + + const res = await ctx.request(`/api/locations/${shelf.id}`, { + method: 'PUT', + cookie: ctx.cookies.admin, + json: { canContainCollections: true, description: 'changed' }, + }) + + expect(res.status).toBe(200) + expect((await loadLocation(shelf.id))?.description).toBe('changed') + }) + }) }) diff --git a/packages/api/src/routes/derivations.ts b/packages/api/src/routes/derivations.ts index 372c8923..b7769dce 100644 --- a/packages/api/src/routes/derivations.ts +++ b/packages/api/src/routes/derivations.ts @@ -9,7 +9,7 @@ import { import { and, eq } from 'drizzle-orm' import { createDerivation } from '../lib/derivations' import { resolveSpecimenSource } from '../lib/specimens/provenance' -import { createDerivationRequestSchema } from '../lib/derivation-schemas' +import { createDerivationRequestSchema, derivationDateSchema } from '../lib/derivation-schemas' import { createAdminMiddleware, createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { requireParam, setsNoField } from '../lib/common-validators' import { handleRouteError, ValidationError } from '../lib/error-handler' @@ -247,7 +247,7 @@ derivations.patch('/derivations/:id', adminMiddleware, async (c) => { const body = await c.req.json() const schema = z.object({ - derivationDate: z.string().optional(), + derivationDate: derivationDateSchema.optional(), protocol: z.string().optional(), notes: z.string().optional(), properties: z.record(z.string(), z.any()).optional(), diff --git a/packages/api/src/routes/imports.ts b/packages/api/src/routes/imports.ts index 065b2ca8..cacc35a6 100644 --- a/packages/api/src/routes/imports.ts +++ b/packages/api/src/routes/imports.ts @@ -5,6 +5,7 @@ import { importDerivationsFromCsv, validateDerivationsCsv, type BulkDerivationSe import { runBulkCombinedImport, validateBulkCombinedPayload } from '../lib/bulk-combined-import' import { createAuthMiddleware, createMemberMiddleware } from '../middleware/auth' import { handleRouteError } from '../lib/error-handler' +import { derivationDateSchema } from '../lib/derivation-schemas' import { bulkCombinedRequestSchema, bulkCombinedValidateRequestSchema, @@ -25,7 +26,8 @@ const bulkDerivationSettingsSchema = z.object({ // '' means the column comes from the CSV per row, like the other settings. containerType: z.enum(['micronix_tube', 'cryovial_tube', 'paper', '']), protocol: z.string(), - derivationDate: z.string(), + // '' means each row gives its own derivation_date. + derivationDate: z.union([z.literal(''), derivationDateSchema]), quantity: z.number().optional(), unitSymbol: z.string().optional(), quantityUsed: z.number().optional(), diff --git a/packages/api/src/routes/locations.ts b/packages/api/src/routes/locations.ts index d5dd3cdd..e6e9f746 100644 --- a/packages/api/src/routes/locations.ts +++ b/packages/api/src/routes/locations.ts @@ -14,8 +14,9 @@ import { updateLocationPath, getLocationStorageTypeId, getLocationHierarchyStats, + countCollectionsAtLocation, } from '../lib/location-helpers' -import { handleRouteError, NotFoundError, ValidationError } from '../lib/error-handler' +import { ConflictError, handleRouteError, NotFoundError, ValidationError } from '../lib/error-handler' import { createAdminMiddleware, createAuthMiddleware } from '../middleware/auth' import { utcNow } from '../lib/datetime' import { requireParam } from '../lib/common-validators' @@ -410,6 +411,16 @@ locations.put('/:id', adminMiddleware, async (c) => { return c.json({ error: 'Location not found' }, 404) } + // A Location that holds Collections must keep holding them (#133). + if (data.canContainCollections === false && existing.canContainCollections) { + const held = await countCollectionsAtLocation(database, id) + if (held > 0) { + throw new ConflictError( + `Cannot turn off "can contain collections": this location holds ${held} ${held === 1 ? 'collection' : 'collections'}. Move them to another location first.`, + ) + } + } + // Validate parent if being changed if (data.parentId !== undefined) { const validationError = await validateLocationHierarchy(database, sqliteDatabase, data.parentId ?? null, id) From 3e7d4cf81c970f68ac742d49286bc59d61dece91 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:18:42 -0800 Subject: [PATCH 39/54] fix(web): number import errors by file row and drop blank Derivations rows Specimen import errors now name the row a spreadsheet shows, counting the header as row 1. Blank rows no longer shift the count, a combined-import error on a specimen grouped under an earlier subject names that specimen's row, and a per-subject import failure names every row of the subject. Derivations import drops rows whose cells are all blank, as the API does, so they no longer appear in review or get submitted, and validation, review and result rows are numbered by their file row. Both parsers share parseCsvFileRows (web lib/csv-file-rows.ts), a small helper that keeps each data row's spreadsheet row number. Closes #138 Co-Authored-By: Claude Opus 5.5 --- .../docs/guides/bulk-operations/import.md | 2 +- .../web/src/components/BulkImportFlow.tsx | 63 ++++++----- .../__tests__/BulkImportFlow.test.tsx | 105 +++++++++++++++++ packages/web/src/hooks/useBulkCsvWorkflow.ts | 17 ++- .../__tests__/derivations-bulk-import.test.ts | 16 ++- packages/web/src/lib/bulk-csv-workflow.ts | 107 +++++++++++++----- packages/web/src/lib/bulk-import-payload.ts | 6 +- .../src/lib/bulk-import-success-message.ts | 2 +- .../web/src/lib/bulk-import-validation.ts | 38 ++++--- packages/web/src/lib/csv-file-rows.ts | 21 ++++ .../__tests__/effects.test.ts | 1 + .../__tests__/reducer.test.ts | 2 + .../src/lib/derivations-bulk-import/csv.ts | 26 +++-- .../lib/derivations-bulk-import/effects.ts | 4 +- .../src/lib/derivations-bulk-import/index.ts | 1 + .../lib/derivations-bulk-import/reducer.ts | 8 ++ .../src/lib/derivations-bulk-import/types.ts | 4 +- .../web/src/pages/DerivationsBulkImport.tsx | 10 +- .../__tests__/DerivationsBulkImport.test.tsx | 74 +++++++++++- 19 files changed, 410 insertions(+), 97 deletions(-) create mode 100644 packages/web/src/lib/csv-file-rows.ts diff --git a/packages/docs/src/content/docs/guides/bulk-operations/import.md b/packages/docs/src/content/docs/guides/bulk-operations/import.md index 6aa18bd1..cec25ae6 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/import.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/import.md @@ -35,7 +35,7 @@ On a study detail page, **Bulk import** already knows the study. For **Subjects Validation has two stages: 1. **Parsing and collection lookup.** The browser maps columns, then asks the server which collections in the file do not exist yet. Row-level rules are not decided in the browser. -2. **Server validation.** Before any write, the server checks study and subject names, specimen types, collection dates, container types and units, that collections exist or have a location so they can be created, that locations can hold collections, and that barcodes and positions are unique in the database and in the file. You see every error at once, with row numbers where applicable. Fix the CSV or collections, then click **Validate & Continue** again. +2. **Server validation.** Before any write, the server checks study and subject names, specimen types, collection dates, container types and units, that collections exist or have a location so they can be created, that locations can hold collections, and that barcodes and positions are unique in the database and in the file. You see every error at once, with the row it is on where applicable. Rows are numbered as a spreadsheet shows them, with the header as row 1, so blank rows count. An error on a subject whose rows are spread through the file names each of its rows. Fix the CSV or collections, then click **Validate & Continue** again. Common failures include missing columns, wrong date or position format, unknown study codes, duplicate barcodes or positions, and locations that cannot hold collections. diff --git a/packages/web/src/components/BulkImportFlow.tsx b/packages/web/src/components/BulkImportFlow.tsx index 24381837..c28f0f58 100644 --- a/packages/web/src/components/BulkImportFlow.tsx +++ b/packages/web/src/components/BulkImportFlow.tsx @@ -14,8 +14,9 @@ import { getBulkImportCollectionType, getBulkImportRequiredFields, getBulkImportOptionalFields, - parseBulkImportCSV, + parseBulkImportCsvFileRows, mapBulkImportRowsToPayload, + formatBulkImportError, getBulkImportRowCollectionRef, type CSVRow, type BulkImportValidationError, @@ -94,11 +95,12 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm created?: number containersCreated?: number combinedSummary?: BulkImportCombinedSummary - errors?: Array<{ index: number; error: string }> + errors?: BulkImportValidationError[] } | null>(null) - const [csvRows, setCsvRows] = useState([]) const [missingCollections, setMissingCollections] = useState([]) const [validatedData, setValidatedData] = useState[]>([]) + // Spreadsheet row of each validatedData item, so errors name the file's row. + const [validatedFileRows, setValidatedFileRows] = useState([]) const [atomicMode, setAtomicMode] = useState('full_file') const bulkCsvWorkflow = useBulkCsvWorkflow({ @@ -130,7 +132,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setValidationErrors([]) setImportResult(null) setCurrentStep('upload') - setCsvRows([]) + setValidatedFileRows([]) setMissingCollections([]) const reader = new FileReader() @@ -162,7 +164,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setPreview([]) setValidationErrors([]) setImportResult(null) - setCsvRows([]) + setValidatedFileRows([]) setMissingCollections([]) const fileInput = document.getElementById('import-csv-file') if (fileInput) { @@ -193,13 +195,14 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm const runPreReviewServerValidation = async ( data: Record[], - missing: MissingCollection[] + fileRows: number[] ): Promise => { - return bulkCsvWorkflow.runServerValidation(data) + return bulkCsvWorkflow.runServerValidation(data, fileRows) } const handleImport = async ( data: Record[], + fileRows: number[], options?: { alreadyValidated?: boolean } ) => { setCurrentStep('import') @@ -208,12 +211,12 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setValidationErrors([]) try { - const result = await bulkCsvWorkflow.runImport(data, { + const result = await bulkCsvWorkflow.runImport(data, fileRows, { skipServerValidate: options?.alreadyValidated, }) if (!result.success && result.errors?.length && result.created == null && !result.combinedSummary) { - setValidationErrors(result.errors.map((e) => ({ row: e.index + 1, error: e.error }))) + setValidationErrors(result.errors) return } @@ -225,14 +228,17 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm errors: result.errors, }) } catch (error: unknown) { - const err = error as { response?: { data?: { error?: string; errors?: Array<{ index: number; error: string }> }; message?: string } } + const err = error as { response?: { data?: { error?: string; errors?: unknown } }; message?: string } const errData = err.response?.data const summaryError = errData?.error || (err as Error).message || 'Import failed' - setValidationErrors([{ row: 0, error: summaryError }]) + setValidationErrors([{ fileRows: [], error: summaryError }]) + const rowErrors = Array.isArray(errData?.errors) + ? bulkCsvWorkflow.importErrorsToFileRows(errData.errors, data, fileRows) + : [] setImportResult({ success: false, created: 0, - errors: Array.isArray(errData?.errors) ? errData.errors : [{ index: 0, error: summaryError }], + errors: rowErrors.length > 0 ? rowErrors : [{ fileRows: [], error: summaryError }], }) } finally { setWorkflowLoading(false) @@ -247,11 +253,11 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm try { const text = await file.text() - const rows = parseBulkImportCSV(text) - setCsvRows(rows) + const { rows, fileRows } = parseBulkImportCsvFileRows(text) + setValidatedFileRows(fileRows) if (rows.length === 0) { - setValidationErrors([{ row: 0, error: 'CSV file is empty or has no data rows' }]) + setValidationErrors([{ fileRows: [], error: 'CSV file is empty or has no data rows' }]) setWorkflowLoading(false) return } @@ -281,7 +287,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm } catch (err: unknown) { setValidationErrors([ { - row: 0, + fileRows: [], error: getQueryErrorMessage(err, 'Failed to check collections'), }, ]) @@ -291,17 +297,17 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm } } - const preErrors = await runPreReviewServerValidation(data, []) + const preErrors = await runPreReviewServerValidation(data, fileRows) if (preErrors.length > 0) { setValidationErrors(preErrors) setWorkflowLoading(false) return } - await handleImport(data, { alreadyValidated: true }) + await handleImport(data, fileRows, { alreadyValidated: true }) } catch (error: unknown) { const message = getQueryErrorMessage(error, 'Validation failed') - setValidationErrors([{ row: 0, error: message }]) + setValidationErrors([{ fileRows: [], error: message }]) } finally { setWorkflowLoading(false) } @@ -370,13 +376,13 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setMissingCollections(updated) if (allSuccess && updated.every(c => c.status === 'success' || c.status === 'pending')) { - const preErrors = await runPreReviewServerValidation(validatedData, updated) + const preErrors = await runPreReviewServerValidation(validatedData, validatedFileRows) if (preErrors.length > 0) { setValidationErrors(preErrors) setWorkflowLoading(false) return } - await handleImport(validatedData, { alreadyValidated: true }) + await handleImport(validatedData, validatedFileRows, { alreadyValidated: true }) } else { setWorkflowLoading(false) } @@ -687,7 +693,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm
    {validationErrors.map((error, i) => (
  • - {error.row > 0 ? `Row ${error.row}: ` : ''}{error.error} + {formatBulkImportError(error)}
    • ))}
@@ -722,8 +728,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm
    {validationErrors.map((error, i) => (
  • - {error.row > 0 ? `Row ${error.row}: ` : ''} - {error.error} + {formatBulkImportError(error)}
    • ))}
@@ -809,13 +814,13 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm try { const preErrors = await runPreReviewServerValidation( validatedData, - missingCollections + validatedFileRows ) if (preErrors.length > 0) { setValidationErrors(preErrors) return } - await handleImport(validatedData, { alreadyValidated: true }) + await handleImport(validatedData, validatedFileRows, { alreadyValidated: true }) } finally { setWorkflowLoading(false) } @@ -848,7 +853,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm
    {validationErrors.map((error, i) => (
  • - {error.row > 0 ? `Row ${error.row}: ` : ''}{error.error} + {formatBulkImportError(error)}
    • ))}
@@ -875,7 +880,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm
    {importResult.errors.map((error, i) => (
  • - Row {error.index + 1}: {error.error} + {formatBulkImportError(error)}
    • ))}
@@ -903,7 +908,7 @@ export default function BulkImportFlow({ fixedStudyShortCode, backLink }: BulkIm setFile(null) setPreview([]) setImportResult(null) - setCsvRows([]) + setValidatedFileRows([]) setMissingCollections([]) setValidatedData([]) }} diff --git a/packages/web/src/components/__tests__/BulkImportFlow.test.tsx b/packages/web/src/components/__tests__/BulkImportFlow.test.tsx index bcc3a8cf..86526fdc 100644 --- a/packages/web/src/components/__tests__/BulkImportFlow.test.tsx +++ b/packages/web/src/components/__tests__/BulkImportFlow.test.tsx @@ -5,6 +5,7 @@ import BulkImportFlow from '../BulkImportFlow' import { importsApi } from '../../lib/api/imports' import { specimenTypesApi } from '../../lib/api/reference-data' import { collectionsApi } from '../../lib/api/collections' +import { specimensApi } from '../../lib/api/specimens' vi.mock('../LocationPicker', () => ({ default: ({ onChange }: { onChange: (id: number | null) => void }) => ( @@ -296,4 +297,108 @@ describe('BulkImportFlow', () => { }) }) }) + + describe('errors name the file row', () => { + /** Upload `text` and validate it; header is row 1, so the file's rows are numbered as a spreadsheet shows them. */ + async function validateCsv( + text: string, + opts: { importType: 'specimens' | 'combined'; containerType: string; perSubject?: boolean }, + ) { + const user = userEvent.setup() + await render(, { initialEntries: ['/import?step=upload'] }) + await user.selectOptions(screen.getByLabelText(/import type/i), opts.importType) + await user.selectOptions(screen.getByLabelText(/container type/i), opts.containerType) + if (opts.perSubject) await user.click(screen.getByRole('radio', { name: /per subject/i })) + const fileInput = document.querySelector('input[type="file"]') as HTMLInputElement + await user.upload(fileInput, new File([text], 'i.csv', { type: 'text/csv' })) + await act(async () => { + fireEvent.submit(fileInput.closest('form')!) + }) + } + + const fileProto = File.prototype as { text?: () => Promise } + const originalText = fileProto.text + beforeAll(() => { + fileProto.text = function (this: File) { + return new Promise((resolve) => { + const reader = new FileReader() + reader.onload = () => resolve(reader.result as string) + reader.readAsText(this) + }) + } + }) + afterAll(() => { + if (originalText) fileProto.text = originalText + else delete fileProto.text + }) + + beforeEach(() => { + // Every plate exists, so validation goes straight to import. + vi.mocked(collectionsApi.check).mockImplementation(({ collections }) => + Promise.resolve({ results: collections.map((c) => ({ ...c, exists: true, id: 1 })) }), + ) + }) + + // Row 1 header; S1 on rows 2 and 5, S2 on row 3, row 4 blank. + const combinedCsv = + 'subject_name,specimen_type_name,plate_name,barcode,position\n' + + 'S1,Serum,P1,MT1,A01\n' + + 'S2,Serum,P1,MT2,A02\n' + + ',,,,\n' + + 'S1,Plasma,P1,MT3,A03\n' + + it('numbers a specimens validation error by its file row when a blank row precedes it', async () => { + vi.mocked(specimensApi.validateBulk).mockResolvedValue({ + valid: false, + errors: [{ index: 1, message: "Subject 'S2' not found" }], + }) + await validateCsv('subject_name,specimen_type_name\nS1,Serum\n\nS2,Plasma\n', { + importType: 'specimens', + containerType: 'none', + }) + + expect(await screen.findByText("Row 4: Subject 'S2' not found")).toBeInTheDocument() + }) + + it('numbers a combined validation error on a specimen grouped under an earlier subject by its file row', async () => { + vi.mocked(importsApi.bulkCombinedValidate).mockImplementation((payload) => + Promise.resolve({ + valid: false, + errors: [ + { + subjectIndex: 0, + specimenIndex: 1, + rowIndex: payload.subjects[0]!.specimens[1]!.rowIndex, + message: 'Position A03 is taken', + }, + ], + }), + ) + await validateCsv(combinedCsv, { importType: 'combined', containerType: 'micronix_tube' }) + + expect(await screen.findByText('Row 5: Position A03 is taken')).toBeInTheDocument() + }) + + it('numbers a combined subject error without a row index by the subject’s first file row', async () => { + vi.mocked(importsApi.bulkCombinedValidate).mockResolvedValue({ + valid: false, + errors: [{ subjectIndex: 1, message: 'Invalid subject name' }], + }) + await validateCsv(combinedCsv, { importType: 'combined', containerType: 'micronix_tube' }) + + expect(await screen.findByText('Row 3: Invalid subject name')).toBeInTheDocument() + }) + + it('names every file row of a subject that failed in a per-subject import', async () => { + vi.mocked(importsApi.bulkCombinedValidate).mockResolvedValue({ valid: true, errors: [] }) + vi.mocked(importsApi.bulkCombined).mockResolvedValue({ + summary: { subjectsCreated: 1, subjectsUpdated: 0, specimensCreated: 1, containersCreated: 1 }, + results: [], + errors: [{ index: 0, error: "Subject 'S1': Position A03 is taken" }], + }) + await validateCsv(combinedCsv, { importType: 'combined', containerType: 'micronix_tube', perSubject: true }) + + expect(await screen.findByText("Rows 2, 5: Subject 'S1': Position A03 is taken")).toBeInTheDocument() + }) + }) }) diff --git a/packages/web/src/hooks/useBulkCsvWorkflow.ts b/packages/web/src/hooks/useBulkCsvWorkflow.ts index 09a8ddc5..46508e81 100644 --- a/packages/web/src/hooks/useBulkCsvWorkflow.ts +++ b/packages/web/src/hooks/useBulkCsvWorkflow.ts @@ -1,6 +1,7 @@ import { useQueryClient } from '@tanstack/react-query' import { invalidateAfterBulkWrite } from '../lib/query-client' import { + bulkCsvImportErrorsToFileRows, runBulkCsvServerValidation, runBulkCsvImport, type BulkCsvWorkflowContext, @@ -8,14 +9,20 @@ import { export type { BulkCsvWorkflowContext, BulkCsvValidationError, BulkCsvSubjectsImportResult } from '../lib/bulk-csv-workflow' -/** Shared bulk CSV workflow hook — parse/validate/import orchestration for import pages. */ -export function useBulkCsvWorkflow(ctx: BulkCsvWorkflowContext) { +/** + * Shared bulk CSV workflow hook — parse/validate/import orchestration for import pages. + * Each call takes the data with its file rows, since both come from the CSV parsed in the same handler. + */ +export function useBulkCsvWorkflow(ctx: Omit) { const queryClient = useQueryClient() return { - runServerValidation: (data: Record[]) => runBulkCsvServerValidation(data, ctx), + runServerValidation: (data: Record[], fileRows: number[]) => + runBulkCsvServerValidation(data, { ...ctx, fileRows }), // Per-subject imports can write partially even when they report errors, so refresh either way. - runImport: (data: Record[], options?: { skipServerValidate?: boolean }) => - runBulkCsvImport(data, ctx, options).finally(() => void invalidateAfterBulkWrite(queryClient)), + runImport: (data: Record[], fileRows: number[], options?: { skipServerValidate?: boolean }) => + runBulkCsvImport(data, { ...ctx, fileRows }, options).finally(() => void invalidateAfterBulkWrite(queryClient)), + importErrorsToFileRows: (errors: unknown[], data: Record[], fileRows: number[]) => + bulkCsvImportErrorsToFileRows(errors, data, { ...ctx, fileRows }), } } diff --git a/packages/web/src/lib/__tests__/derivations-bulk-import.test.ts b/packages/web/src/lib/__tests__/derivations-bulk-import.test.ts index 33f59513..04c3cb0c 100644 --- a/packages/web/src/lib/__tests__/derivations-bulk-import.test.ts +++ b/packages/web/src/lib/__tests__/derivations-bulk-import.test.ts @@ -20,8 +20,20 @@ const emptySettings: BulkDerivationSettings = { describe('parseFullCsv', () => { it('returns empty for header-only or empty text', () => { - expect(parseFullCsv('')).toEqual({ headers: [], rows: [] }) - expect(parseFullCsv('a,b')).toEqual({ headers: [], rows: [] }) + expect(parseFullCsv('')).toEqual({ headers: [], rows: [], fileRows: [] }) + expect(parseFullCsv('a,b')).toEqual({ headers: [], rows: [], fileRows: [] }) + expect(parseFullCsv('a,b\n,\n\n')).toEqual({ headers: [], rows: [], fileRows: [] }) + }) + + it('drops rows whose cells are all blank and keeps the file row of the rest', () => { + const parsed = parseFullCsv('\na,b\n1,2\n\n , \n"x\ny",4\n5,6\n') + expect(parsed.rows).toEqual([ + { a: '1', b: '2' }, + { a: 'x\ny', b: '4' }, + { a: '5', b: '6' }, + ]) + // Rows as a spreadsheet numbers them: a quoted line break stays inside one row. + expect(parsed.fileRows).toEqual([3, 6, 7]) }) it('parses headers and rows, trimming values', () => { diff --git a/packages/web/src/lib/bulk-csv-workflow.ts b/packages/web/src/lib/bulk-csv-workflow.ts index e64a7f16..7b22b80b 100644 --- a/packages/web/src/lib/bulk-csv-workflow.ts +++ b/packages/web/src/lib/bulk-csv-workflow.ts @@ -14,9 +14,9 @@ import { } from './bulk-import-payload' import type { MissingCollectionForPayload } from './bulk-import-payload' import type { ContainerType } from './container-types' -import type { ImportType } from './bulk-import-validation' +import type { BulkImportValidationError, ImportType } from './bulk-import-validation' -export type BulkCsvValidationError = { row: number; error: string } +export type BulkCsvValidationError = BulkImportValidationError export type BulkCsvSubjectsImportResult = { success: boolean @@ -28,7 +28,7 @@ export type BulkCsvSubjectsImportResult = { specimensCreated: number containersCreated: number } - errors?: Array<{ index: number; error: string }> + errors?: BulkCsvValidationError[] } export type BulkCsvWorkflowContext = { @@ -37,6 +37,71 @@ export type BulkCsvWorkflowContext = { fixedStudyShortCode?: string missingCollections: MissingCollectionForPayload[] atomicMode: BulkCombinedAtomicMode + /** Spreadsheet row of each data item, counting the header as row 1. */ + fileRows: number[] +} + +type CombinedPayload = ReturnType + +function combinedPayload(data: Record[], ctx: BulkCsvWorkflowContext): CombinedPayload { + return buildBulkCombinedRequestPayload(data, { + containerType: ctx.containerType, + fixedStudyShortCode: ctx.fixedStudyShortCode, + missingCollections: ctx.missingCollections, + atomicMode: ctx.atomicMode, + fileRows: ctx.fileRows, + }) +} + +/** The spreadsheet row of data item `index`. */ +function fileRowOf(ctx: BulkCsvWorkflowContext, index: number): number { + return ctx.fileRows[index] ?? index + 2 +} + +/** Spreadsheet rows of a subject's specimens: the payload groups rows by subject, so they need not be adjacent. */ +function subjectFileRows(payload: CombinedPayload, subjectIndex: number): number[] { + return (payload.subjects[subjectIndex]?.specimens ?? []).flatMap((s) => (s.rowIndex == null ? [] : [s.rowIndex])) +} + +/** A combined-import error the API reports per subject, and per specimen when it knows which. */ +type CombinedSubjectError = { subjectIndex: number; specimenIndex?: number; rowIndex?: number; message: string } + +function combinedErrorFileRows(payload: CombinedPayload, e: CombinedSubjectError): number[] { + if (e.rowIndex != null) return [e.rowIndex] + const rows = subjectFileRows(payload, e.subjectIndex) + const specimenRow = e.specimenIndex == null ? undefined : rows[e.specimenIndex] + if (specimenRow != null) return [specimenRow] + return rows.slice(0, 1) +} + +function isCombinedSubjectError(e: unknown): e is CombinedSubjectError { + return typeof e === 'object' && e != null && typeof (e as CombinedSubjectError).subjectIndex === 'number' +} + +function isIndexedError(e: unknown): e is { index: number; error: string } { + return typeof e === 'object' && e != null && typeof (e as { index?: unknown }).index === 'number' +} + +/** + * Map the API's per-item import errors to spreadsheet rows. `index` names a data item, except in + * a combined import, where it names a subject and so every row of that subject. + */ +export function bulkCsvImportErrorsToFileRows( + errors: unknown[], + data: Record[], + ctx: BulkCsvWorkflowContext, +): BulkCsvValidationError[] { + const payload = ctx.importType === 'combined' ? combinedPayload(data, ctx) : null + return errors.flatMap((e): BulkCsvValidationError[] => { + if (payload && isCombinedSubjectError(e)) { + return [{ fileRows: combinedErrorFileRows(payload, e), error: e.message }] + } + if (isIndexedError(e)) { + const fileRows = payload ? subjectFileRows(payload, e.index) : [fileRowOf(ctx, e.index)] + return [{ fileRows, error: e.error }] + } + return [] + }) } export async function runBulkCsvServerValidation( @@ -48,7 +113,7 @@ export async function runBulkCsvServerValidation( subjects: data as Array<{ studyShortCode: string; name: string }>, }) if (validateRes.valid || validateRes.errors.length === 0) return [] - return validateRes.errors.map((e) => ({ row: e.index + 1, error: e.message })) + return validateRes.errors.map((e) => ({ fileRows: [fileRowOf(ctx, e.index)], error: e.message })) } if (ctx.importType === 'specimens') { @@ -59,21 +124,13 @@ export async function runBulkCsvServerValidation( specimens: specimensWithLocations as SpecimenBulkItem[], }) if (validateRes.valid) return [] - return validateRes.errors.map((e) => ({ row: e.index + 1, error: e.message })) + return validateRes.errors.map((e) => ({ fileRows: [fileRowOf(ctx, e.index)], error: e.message })) } - const payload = buildBulkCombinedRequestPayload(data, { - containerType: ctx.containerType, - fixedStudyShortCode: ctx.fixedStudyShortCode, - missingCollections: ctx.missingCollections, - atomicMode: ctx.atomicMode, - }) + const payload = combinedPayload(data, ctx) const validateRes = await importsApi.bulkCombinedValidate(payload) if (validateRes.valid || validateRes.errors.length === 0) return [] - return validateRes.errors.map((e) => ({ - row: e.rowIndex ?? e.subjectIndex + 1, - error: e.message, - })) + return validateRes.errors.map((e) => ({ fileRows: combinedErrorFileRows(payload, e), error: e.message })) } export async function runBulkCsvImport( @@ -84,7 +141,7 @@ export async function runBulkCsvImport( if (!options?.skipServerValidate) { const errors = await runBulkCsvServerValidation(data, ctx) if (errors.length > 0) { - return { success: false, errors: errors.map((e, i) => ({ index: i, error: e.error })) } + return { success: false, errors } } } @@ -92,7 +149,11 @@ export async function runBulkCsvImport( const response = await subjectsApi.createBulk({ subjects: data as Array<{ studyShortCode: string; name: string }>, }) - return { success: true, created: response.created, errors: response.errors } + return { + success: true, + created: response.created, + errors: response.errors && bulkCsvImportErrorsToFileRows(response.errors, data, ctx), + } } if (ctx.importType === 'specimens') { @@ -106,20 +167,14 @@ export async function runBulkCsvImport( success: true, created: response.created, containersCreated: response.containersCreated, - errors: response.errors, + errors: response.errors && bulkCsvImportErrorsToFileRows(response.errors, data, ctx), } } - const validatePayload = buildBulkCombinedRequestPayload(data, { - containerType: ctx.containerType, - fixedStudyShortCode: ctx.fixedStudyShortCode, - missingCollections: ctx.missingCollections, - atomicMode: ctx.atomicMode, - }) - const response = await importsApi.bulkCombined(toBulkCombinedImportRequest(validatePayload)) + const response = await importsApi.bulkCombined(toBulkCombinedImportRequest(combinedPayload(data, ctx))) return { success: !response.errors?.length, combinedSummary: response.summary, - errors: response.errors, + errors: response.errors && bulkCsvImportErrorsToFileRows(response.errors, data, ctx), } } diff --git a/packages/web/src/lib/bulk-import-payload.ts b/packages/web/src/lib/bulk-import-payload.ts index bd828392..880e0bdc 100644 --- a/packages/web/src/lib/bulk-import-payload.ts +++ b/packages/web/src/lib/bulk-import-payload.ts @@ -175,7 +175,7 @@ export function toBulkCombinedImportRequest( } } -/** Combined import validate payload (includes rowIndex for CSV alignment). */ +/** Combined import validate payload (includes rowIndex, the file row, for CSV alignment). */ export function buildBulkCombinedRequestPayload( data: Record[], opts: { @@ -184,6 +184,8 @@ export function buildBulkCombinedRequestPayload( /** Pass component state: missing collections with locations for full_file create. */ missingCollections: MissingCollectionForPayload[] atomicMode: BulkCombinedAtomicMode + /** Spreadsheet row of each `data` item; sent as rowIndex so errors name the file row. */ + fileRows?: number[] } ): BulkCombinedValidateRequest { const { fixedStudyShortCode, missingCollections } = opts @@ -213,7 +215,7 @@ export function buildBulkCombinedRequestPayload( specimenTypeName: spec.specimenTypeName as string, collectionDate: spec.collectionDate as string | undefined, container: containerData, - rowIndex: rowIndex + 1, + rowIndex: opts.fileRows?.[rowIndex] ?? rowIndex + 1, }) } diff --git a/packages/web/src/lib/bulk-import-success-message.ts b/packages/web/src/lib/bulk-import-success-message.ts index 160a76ed..0f186d94 100644 --- a/packages/web/src/lib/bulk-import-success-message.ts +++ b/packages/web/src/lib/bulk-import-success-message.ts @@ -15,7 +15,7 @@ export type BulkImportResultForMessage = { created?: number containersCreated?: number combinedSummary?: BulkImportCombinedSummary - errors?: Array<{ index: number; error: string }> + errors?: ReadonlyArray } function plural(n: number, one: string, many: string): string { diff --git a/packages/web/src/lib/bulk-import-validation.ts b/packages/web/src/lib/bulk-import-validation.ts index bbc85a94..56929b0f 100644 --- a/packages/web/src/lib/bulk-import-validation.ts +++ b/packages/web/src/lib/bulk-import-validation.ts @@ -5,17 +5,24 @@ import type { ContainerType } from './container-types' import { getCollectionNameColumn } from './container-columns' import type { FlatBulkImportContainer } from './bulk-import-payload' -import { parseCsv } from '@sampledb/contract' +import { parseCsvFileRows } from './csv-file-rows' export interface CSVRow { [key: string]: string } +/** An import error and the spreadsheet rows it concerns (header is row 1); none for a file-wide error. */ export interface BulkImportValidationError { - row: number + fileRows: number[] error: string } +/** "Row 3: …", "Rows 2, 5: …", or the bare message for a file-wide error. */ +export function formatBulkImportError({ fileRows, error }: BulkImportValidationError): string { + if (fileRows.length === 0) return error + return `${fileRows.length === 1 ? 'Row' : 'Rows'} ${fileRows.join(', ')}: ${error}` +} + export type ImportType = 'subjects' | 'specimens' | 'combined' export type BulkImportCollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' @@ -85,22 +92,27 @@ function normalizeHeader(header: string): string { } export function parseBulkImportCSV(text: string): CSVRow[] { - // Shared RFC 4180 parser: quoted commas ("low vol, hemolyzed"), quotes, CRLF, and BOM. - const lines = parseCsv(text, { skipBlankRows: true }) - if (lines.length < 2) return [] + return parseBulkImportCsvFileRows(text).rows +} - const headers = lines[0].map((h) => normalizeHeader(h.trim())) - const rows: CSVRow[] = [] +/** + * Parse bulk import CSV into header-keyed rows, dropping rows whose cells are all blank. + * `fileRows[i]` is the spreadsheet row of `rows[i]`, counting the header as row 1. + */ +export function parseBulkImportCsvFileRows(text: string): { rows: CSVRow[]; fileRows: number[] } { + // Shared RFC 4180 parser: quoted commas ("low vol, hemolyzed"), quotes, CRLF, and BOM. + const parsed = parseCsvFileRows(text) + if (!parsed) return { rows: [], fileRows: [] } - for (let i = 1; i < lines.length; i++) { - const values = lines[i] + const headers = parsed.header.map((h) => normalizeHeader(h.trim())) + const rows = parsed.rows.map(({ cells }) => { const row: CSVRow = {} headers.forEach((header, j) => { - row[header] = values[j]?.trim() ?? '' + row[header] = cells[j]?.trim() ?? '' }) - rows.push(row) - } - return rows + return row + }) + return { rows, fileRows: parsed.rows.map((r) => r.fileRow) } } /** Resolve paper parent from bulk import CSV row (exactly one of box_name or bag_name). */ diff --git a/packages/web/src/lib/csv-file-rows.ts b/packages/web/src/lib/csv-file-rows.ts new file mode 100644 index 00000000..c4614e97 --- /dev/null +++ b/packages/web/src/lib/csv-file-rows.ts @@ -0,0 +1,21 @@ +import { parseCsv } from '@sampledb/contract' + +/** A CSV data row and the row a spreadsheet shows it on, counting the header as row 1. */ +export interface CsvFileRow { + cells: string[] + fileRow: number +} + +/** + * Parse CSV text into its header and data rows, dropping rows whose cells are all blank. + * Each data row keeps its spreadsheet row number, so dropped rows don't shift the rows after them. + */ +export function parseCsvFileRows(text: string): { header: string[]; rows: CsvFileRow[] } | null { + const records = parseCsv(text) + const filled = records.flatMap((cells, i) => + cells.every((cell) => cell.trim() === '') ? [] : [{ cells, fileRow: i + 1 }], + ) + if (filled.length === 0) return null + const [header, ...rows] = filled + return { header: header.cells, rows } +} diff --git a/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts b/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts index f143af32..25c1ade6 100644 --- a/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts +++ b/packages/web/src/lib/derivations-bulk-import/__tests__/effects.test.ts @@ -52,6 +52,7 @@ describe('validateDerivationsCsv', () => { result: emptyValidation(), headers: ['a', 'b'], rows: [{ a: '1', b: '2' }], + fileRows: [2], }, { type: 'STEP_SET', step: 'review' }, ]) diff --git a/packages/web/src/lib/derivations-bulk-import/__tests__/reducer.test.ts b/packages/web/src/lib/derivations-bulk-import/__tests__/reducer.test.ts index d51df7df..036e3b17 100644 --- a/packages/web/src/lib/derivations-bulk-import/__tests__/reducer.test.ts +++ b/packages/web/src/lib/derivations-bulk-import/__tests__/reducer.test.ts @@ -45,6 +45,7 @@ describe('derivationsBulkReducer', () => { result, headers: ['a', 'b'], rows: [{ a: '1', b: '2' }], + fileRows: [2], }) expect(nextStepAfterValidate(result)).toBe('collections') expect(selectMissingCollections(validated)).toEqual([ @@ -65,6 +66,7 @@ describe('derivationsBulkReducer', () => { result, headers: ['a'], rows: [{ a: '1' }], + fileRows: [2], }) state = derivationsBulkReducer(state, { type: 'COLLECTION_PATCHED', diff --git a/packages/web/src/lib/derivations-bulk-import/csv.ts b/packages/web/src/lib/derivations-bulk-import/csv.ts index b23e3a48..87a23978 100644 --- a/packages/web/src/lib/derivations-bulk-import/csv.ts +++ b/packages/web/src/lib/derivations-bulk-import/csv.ts @@ -1,18 +1,26 @@ -import { parseCsv, serializeCsv } from '@sampledb/contract' +import { serializeCsv } from '@sampledb/contract' +import { parseCsvFileRows } from '../csv-file-rows' -/** Parse CSV text into header-keyed rows via the shared contract parser. */ -export function parseFullCsv(csv: string): { headers: string[]; rows: Record[] } { - const cells = parseCsv(csv).filter(row => !(row.length === 1 && row[0].trim() === '')) - if (cells.length < 2) return { headers: [], rows: [] } - const headers = cells[0].map(h => h.trim()) - const rows = cells.slice(1).map(cols => { +/** + * Parse CSV text into header-keyed rows, dropping rows whose cells are all blank as the + * API does. `fileRows[i]` is the spreadsheet row of `rows[i]`, counting the header as row 1. + */ +export function parseFullCsv(csv: string): { + headers: string[] + rows: Record[] + fileRows: number[] +} { + const parsed = parseCsvFileRows(csv) + if (!parsed || parsed.rows.length === 0) return { headers: [], rows: [], fileRows: [] } + const headers = parsed.header.map(h => h.trim()) + const rows = parsed.rows.map(({ cells }) => { const row: Record = {} headers.forEach((header, j) => { - row[header] = cols[j]?.trim() ?? '' + row[header] = cells[j]?.trim() ?? '' }) return row }) - return { headers, rows } + return { headers, rows, fileRows: parsed.rows.map(r => r.fileRow) } } /** First few data rows for the upload-step preview table. */ diff --git a/packages/web/src/lib/derivations-bulk-import/effects.ts b/packages/web/src/lib/derivations-bulk-import/effects.ts index e4dcbc43..595281cf 100644 --- a/packages/web/src/lib/derivations-bulk-import/effects.ts +++ b/packages/web/src/lib/derivations-bulk-import/effects.ts @@ -26,9 +26,9 @@ export async function validateDerivationsCsv( } try { const result = await gateway.validateCsv(csvContent, settings) - const { headers, rows } = parseFullCsv(csvContent) + const { headers, rows, fileRows } = parseFullCsv(csvContent) return [ - { type: 'VALIDATED', result, headers, rows: rows.map((row) => ({ ...row })) }, + { type: 'VALIDATED', result, headers, rows: rows.map((row) => ({ ...row })), fileRows }, { type: 'STEP_SET', step: nextStepAfterValidate(result) }, ] } catch (error: unknown) { diff --git a/packages/web/src/lib/derivations-bulk-import/index.ts b/packages/web/src/lib/derivations-bulk-import/index.ts index 4f4b6df6..d1638126 100644 --- a/packages/web/src/lib/derivations-bulk-import/index.ts +++ b/packages/web/src/lib/derivations-bulk-import/index.ts @@ -17,6 +17,7 @@ export { derivationsBulkReducer, selectMissingCollections, selectCsvForImport, + selectFileRow, } from './reducer' export { fileLoadedEvent, diff --git a/packages/web/src/lib/derivations-bulk-import/reducer.ts b/packages/web/src/lib/derivations-bulk-import/reducer.ts index 1a75269c..adcca0b6 100644 --- a/packages/web/src/lib/derivations-bulk-import/reducer.ts +++ b/packages/web/src/lib/derivations-bulk-import/reducer.ts @@ -8,6 +8,7 @@ export function initialDerivationsBulkState(): DerivationsBulkState { csvContent: '', reviewHeaders: [], reviewRows: [], + reviewFileRows: [], validationResult: null, importResults: null, error: null, @@ -35,6 +36,7 @@ export function derivationsBulkReducer( importResults: null, reviewHeaders: [], reviewRows: [], + reviewFileRows: [], collectionUpdates: {}, error: null, step: 'upload', @@ -55,6 +57,7 @@ export function derivationsBulkReducer( validationResult: event.result, reviewHeaders: event.headers, reviewRows: event.rows.map((row) => ({ ...row })), + reviewFileRows: [...event.fileRows], collectionUpdates: {}, error: null, importResults: null, @@ -98,3 +101,8 @@ export function selectMissingCollections(state: DerivationsBulkState): MissingDe export function selectCsvForImport(state: DerivationsBulkState): string { return csvForImport(state.csvContent, state.reviewHeaders, state.reviewRows) } + +/** The spreadsheet row of the API's row `index`, counting the header as row 1. */ +export function selectFileRow(state: DerivationsBulkState, index: number): number { + return state.reviewFileRows[index] ?? index + 2 +} diff --git a/packages/web/src/lib/derivations-bulk-import/types.ts b/packages/web/src/lib/derivations-bulk-import/types.ts index 2f17068a..14d0258d 100644 --- a/packages/web/src/lib/derivations-bulk-import/types.ts +++ b/packages/web/src/lib/derivations-bulk-import/types.ts @@ -22,6 +22,8 @@ export interface DerivationsBulkState { csvContent: string reviewHeaders: string[] reviewRows: Record[] + /** Spreadsheet row of each review row (header is row 1); the API's row index points into this. */ + reviewFileRows: number[] validationResult: ValidationResult | null importResults: DerivationCsvImportResultRow[] | null error: string | null @@ -33,7 +35,7 @@ export type DerivationsBulkEvent = | { type: 'FILE_CLEARED' } | { type: 'ERROR'; message: string } | { type: 'ERROR_CLEARED' } - | { type: 'VALIDATED'; result: ValidationResult; headers: string[]; rows: Record[] } + | { type: 'VALIDATED'; result: ValidationResult; headers: string[]; rows: Record[]; fileRows: number[] } | { type: 'STEP_SET'; step: DerivationsBulkStep } | { type: 'REVIEW_CELL_SET'; rowIndex: number; header: string; value: string } | { type: 'COLLECTION_PATCHED'; index: number; patch: Partial } diff --git a/packages/web/src/pages/DerivationsBulkImport.tsx b/packages/web/src/pages/DerivationsBulkImport.tsx index 275a799f..cb3447f5 100644 --- a/packages/web/src/pages/DerivationsBulkImport.tsx +++ b/packages/web/src/pages/DerivationsBulkImport.tsx @@ -14,6 +14,7 @@ import { getRequiredAndOptionalColumns, parseCsvPreview, resolveTemplateParentType, + selectFileRow, type ParentContainerType, type SourceType, } from '../lib/derivations-bulk-import' @@ -79,6 +80,7 @@ export default function DerivationsBulkImport() { const { specimenTypes, units, allowedContainerTypes, bootstrapLoading, bootstrapError, containerTypesError } = bootstrap const formDisabled = workflowLoading || bootstrapLoading + const fileRow = (index: number) => selectFileRow(state, index) const invalidValidationRows = validationResult?.rows.filter((row) => !row.valid && row.error) ?? [] const fileInputRef = useRef(null) @@ -776,7 +778,7 @@ export default function DerivationsBulkImport() {
    {invalidValidationRows.map((row) => (
  • - Row {row.index + 1}: {row.error} + Row {fileRow(row.index)}: {row.error}
    • ))}
@@ -797,7 +799,7 @@ export default function DerivationsBulkImport() {
{rowIndex + 1}{fileRow(rowIndex)} - {row.index + 1} + {fileRow(row.index)} {row.valid ? ( @@ -1081,7 +1083,7 @@ export default function DerivationsBulkImport() { className={row.success ? 'bg-app-trend-up/10' : 'bg-app-trend-down/10'} > - {row.index + 1} + {fileRow(row.index)} {row.success ? ( diff --git a/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx b/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx index e619b70c..9f2b516b 100644 --- a/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx +++ b/packages/web/src/pages/__tests__/DerivationsBulkImport.test.tsx @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import userEvent from '@testing-library/user-event' -import { render, screen, waitFor } from '../../__tests__/helpers/render' +import { render, screen, waitFor, within } from '../../__tests__/helpers/render' import DerivationsBulkImport from '../DerivationsBulkImport' import { specimenTypesApi, unitsApi } from '../../lib/api/reference-data' import { derivationsApi } from '../../lib/api/derivations' @@ -102,6 +102,76 @@ describe('DerivationsBulkImport', () => { await waitFor(() => expect(validate).toBeEnabled()) await user.click(validate) - expect(await screen.findByRole('alert')).toHaveTextContent(`Row 2: ${message}`) + // The file's row 3, counting the header as row 1. + expect(await screen.findByRole('alert')).toHaveTextContent(`Row 3: ${message}`) + }) + + describe('a file with blank rows', () => { + // Row 1 header, row 2 data, rows 3 and 4 blank, row 5 data, row 6 blank. + const csv = + 'parent_container_id,plate_name,position,container_barcode\n' + + '1,P,A01,C1\n' + + '\n' + + ',,,\n' + + '2,P,A02,C2\n' + + ', ,,\n' + + async function uploadAndValidate() { + const user = userEvent.setup() + const { container } = await render() + const fileInput = container.querySelector('#derivation-csv-file')! + await waitFor(() => expect(fileInput).toBeEnabled()) + await user.upload(fileInput, new File([csv], 'derivations.csv', { type: 'text/csv' })) + const validate = screen.getByRole('button', { name: /validate & continue/i }) + await waitFor(() => expect(validate).toBeEnabled()) + await user.click(validate) + return user + } + + it('leaves blank rows out of review and numbers the rest by their file row', async () => { + vi.mocked(derivationsApi.validateCsv).mockResolvedValue({ + rows: [ + { index: 0, valid: true }, + { index: 1, valid: false, error: 'Parent container not found' }, + ], + collections: [], + summary: { total: 2, valid: 1, invalid: 1, warnings: 0 }, + }) + await uploadAndValidate() + + expect(await screen.findByRole('alert')).toHaveTextContent('Row 5: Parent container not found') + const tables = screen.getAllByRole('table') + const reviewTable = tables[tables.length - 1]! + const bodyRows = within(reviewTable).getAllByRole('row').slice(1) + expect(bodyRows.map((row) => within(row).getAllByRole('cell')[0]!.textContent)).toEqual(['2', '5']) + }) + + it('submits no blank rows and numbers import results by their file row', async () => { + vi.mocked(derivationsApi.validateCsv).mockResolvedValue({ + rows: [ + { index: 0, valid: true }, + { index: 1, valid: true }, + ], + collections: [], + summary: { total: 2, valid: 2, invalid: 0, warnings: 0 }, + }) + vi.mocked(derivationsApi.importCsv).mockResolvedValue({ + rows: [ + { index: 0, success: true }, + { index: 1, success: false, error: 'Position A02 is taken' }, + ], + }) + const user = await uploadAndValidate() + await user.click(await screen.findByRole('button', { name: /create derivations/i })) + + await waitFor(() => expect(derivationsApi.importCsv).toHaveBeenCalled()) + const sent = vi.mocked(derivationsApi.importCsv).mock.calls[0]![0] + expect(sent).toBe('parent_container_id,plate_name,position,container_barcode\n1,P,A01,C1\n2,P,A02,C2') + + const results = await screen.findByRole('heading', { name: 'Import results' }) + const resultsTable = within(results.parentElement!).getByRole('table') + const resultRows = within(resultsTable).getAllByRole('row').slice(1) + expect(resultRows.map((row) => within(row).getAllByRole('cell')[0]!.textContent)).toEqual(['2', '5']) + }) }) }) From 683a0e25ff9129c7d3c79acc3c5136c389c4688d Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:19:31 -0800 Subject: [PATCH 40/54] fix(subjects): search Merge Subjects on the server; create a new subject with its specimen Merge Subjects loaded one page of up to 1,000 subjects and filtered it in the browser, so a subject past that could not be picked. Each picker now searches GET /studies/:id/subjects with a new `search` parameter (name contains the text, taken literally, or the exact subject ID) and lists the first 50 matches, with their specimen counts from the same response. Specimen registration with "Create new subject" created the subject, then the specimen; a failed specimen (e.g. a duplicate barcode) left the subject behind and a retry failed on it. The form now sends both in one request to POST /subjects/with-specimens, which writes them in one transaction, and shows the API's error. Also corrects the web type of that endpoint's response, whose specimens are flat. Fixes #135 Co-Authored-By: Claude Opus 5.5 --- .../api/src/routes/__tests__/studies.test.ts | 59 +- packages/api/src/routes/studies.ts | 13 +- .../guides/workflows/subjects-specimens.md | 2 +- .../__tests__/helpers/mock-api-templates.ts | 2 +- .../web/src/components/SubjectMergeModal.tsx | 561 +++++++----------- .../__tests__/SubjectMergeModal.test.tsx | 128 ++++ .../web/src/components/forms/SpecimenForm.tsx | 63 +- .../forms/__tests__/SpecimenForm.test.tsx | 63 +- .../src/hooks/__tests__/useSpecimens.test.tsx | 60 +- packages/web/src/hooks/useSpecimens.ts | 50 ++ packages/web/src/hooks/useStudies.ts | 12 +- packages/web/src/lib/api/studies.ts | 2 +- packages/web/src/lib/api/subjects.ts | 6 +- 13 files changed, 622 insertions(+), 399 deletions(-) create mode 100644 packages/web/src/components/__tests__/SubjectMergeModal.test.tsx diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index d369ebb0..58de03d7 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -5,7 +5,7 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createStudiesRoutes } from '../studies' -import { createTestStudy, createTestSpecimenType, createTestUnit } from '../../__tests__/helpers/factories' +import { createTestStudy, createTestSpecimenType, createTestUnit, createTestStudySubject } from '../../__tests__/helpers/factories' describe('Studies API (list, get, create, update)', () => { let ctx: AuthenticatedRouteTestContext @@ -204,6 +204,63 @@ describe('Studies API (list, get, create, update)', () => { }) }) + describe('GET /api/studies/:id/subjects search', () => { + type SubjectsBody = { subjects: { id: number; name: string }[]; pagination: { total: number } } + + it('finds a subject past the first 1,000 by name', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Big Study', shortCode: 'BIG02' }) + const addSubject = ctx.sqlite.prepare( + "INSERT INTO study_subject (study_id, name, created, last_updated) VALUES (?, ?, '2024-01-01', '2024-01-01')", + ) + ctx.sqlite.exec('BEGIN') + for (let i = 0; i < 1_100; i++) addSubject.run(studyRecord.id, `P${String(i).padStart(5, '0')}`) + ctx.sqlite.exec('COMMIT') + + const res = await ctx.request(`/api/studies/${studyRecord.id}/subjects?search=P01050&page=1&limit=20`) + + expect(res.status).toBe(200) + const data = (await res.json()) as SubjectsBody + expect(data.subjects.map((s) => s.name)).toEqual(['P01050']) + expect(data.pagination.total).toBe(1) + }) + + it('matches a subject by its exact ID', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Study', shortCode: 'IDS01' }) + const target = await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'Alpha' }) + await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'Beta' }) + + const res = await ctx.request(`/api/studies/${studyRecord.id}/subjects?search=${target.id}&page=1&limit=20`) + + expect(res.status).toBe(200) + const data = (await res.json()) as SubjectsBody + expect(data.subjects.map((s) => s.id)).toEqual([target.id]) + }) + + it('takes % and _ in the search literally', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Study', shortCode: 'LIT01' }) + await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'A_1' }) + await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'AB1' }) + + const res = await ctx.request(`/api/studies/${studyRecord.id}/subjects?search=${encodeURIComponent('A_')}&page=1&limit=20`) + + const data = (await res.json()) as SubjectsBody + expect(data.subjects.map((s) => s.name)).toEqual(['A_1']) + }) + + it('keeps only subjects of the requested study', async () => { + const first = await createTestStudy(ctx.db, { title: 'First', shortCode: 'ONE01' }) + const second = await createTestStudy(ctx.db, { title: 'Second', shortCode: 'TWO01' }) + await createTestStudySubject(ctx.db, { studyId: first.id, name: 'Shared-1' }) + await createTestStudySubject(ctx.db, { studyId: second.id, name: 'Shared-2' }) + + const res = await ctx.request(`/api/studies/${first.id}/subjects?search=Shared&page=1&limit=20`) + + const data = (await res.json()) as SubjectsBody + expect(data.subjects.map((s) => s.name)).toEqual(['Shared-1']) + expect(data.pagination.total).toBe(1) + }) + }) + describe('GET /api/studies/:id/summary', () => { it('handles studies with more specimens than SQLite allows as bound parameters', async () => { const studyRecord = await createTestStudy(ctx.db, { title: 'Big Study', shortCode: 'BIG01' }) diff --git a/packages/api/src/routes/studies.ts b/packages/api/src/routes/studies.ts index 7500e471..c22a685b 100644 --- a/packages/api/src/routes/studies.ts +++ b/packages/api/src/routes/studies.ts @@ -26,6 +26,7 @@ import { withWriteTransaction } from '../db/write-transaction' import { assertNotUsedInQpcr } from '../lib/qpcr-usage' import { insertedRow } from '../lib/inserted-row' import { timestampFilter } from '../lib/date-filters' +import { likeContains } from '../lib/sql-like' /** Short code prefix for tutorial namespace. Any study whose short code starts with this (case-insensitive) may be deleted by any member. Only admins may rename a study into or out of this namespace. */ const TUTORIAL_SHORT_CODE_PREFIX = 'TUT' @@ -146,8 +147,16 @@ studies.get('/:id/subjects', authMiddleware, async (c) => { } const offset = returnAll ? undefined : (page - 1) * limit! - const whereClause = eq(studySubject.studyId, id) - + // search matches a subject name containing the text, or the exact subject ID. + const search = c.req.query('search')?.trim() + const searchId = search && /^\d+$/.test(search) ? Number(search) : undefined + const whereClause = and( + eq(studySubject.studyId, id), + search + ? or(likeContains(studySubject.name, search), searchId !== undefined ? eq(studySubject.id, searchId) : undefined) + : undefined, + ) + // Get all subjects or paginated subset let query = database .select({ diff --git a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md index b60185cf..b6c28583 100644 --- a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md +++ b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md @@ -29,7 +29,7 @@ Open the form from any of these places: ### Specimen fields 1. Select a source type. **Subject** is the usual choice. The other options are **Control**, **Reagent**, **Cell Line**, **Plasmid**, and **Standard**. -2. If the source is **Subject**, select the study or enter its short code, then select the subject. To create the subject in the same step, select the **Create New Subject** checkbox and enter the name. +2. If the source is **Subject**, select the study or enter its short code, then select the subject. To create the subject in the same step, select the **Create New Subject** checkbox and enter the name. The subject and specimen are saved together: if the specimen can't be saved, for example because a barcode is already in use, the subject isn't created either, and the error says why. If the study already has a subject with that name, the specimen is added to it. 3. Select a **Specimen Type** from Reference Data. 4. Optional: enter **Collection Date** as YYYY-MM-DD. If you leave it blank, the specimen is saved without a date. SampleDB does not default to today. diff --git a/packages/web/src/__tests__/helpers/mock-api-templates.ts b/packages/web/src/__tests__/helpers/mock-api-templates.ts index d5abb739..04559ffe 100644 --- a/packages/web/src/__tests__/helpers/mock-api-templates.ts +++ b/packages/web/src/__tests__/helpers/mock-api-templates.ts @@ -442,7 +442,7 @@ export function specimenFormMock(): MockApiOverrides { cellLinesApi: { list: vi.fn().mockResolvedValue({ cellLines: [] }) }, plasmidsApi: { list: vi.fn().mockResolvedValue({ plasmids: [] }) }, standardsApi: { list: vi.fn().mockResolvedValue({ standards: [] }) }, - subjectsApi: { create: vi.fn().mockResolvedValue({ subject: { id: 1 } }) }, + subjectsApi: { create: vi.fn().mockResolvedValue({ subject: { id: 1 } }), createWithSpecimens: vi.fn() }, settingsApi: { getContainerTypeUnits: vi.fn().mockResolvedValue({ units: [] }), get: vi.fn().mockResolvedValue({ key: 'container_defaults', value: null }), diff --git a/packages/web/src/components/SubjectMergeModal.tsx b/packages/web/src/components/SubjectMergeModal.tsx index b1a8d87b..1fa28600 100644 --- a/packages/web/src/components/SubjectMergeModal.tsx +++ b/packages/web/src/components/SubjectMergeModal.tsx @@ -1,12 +1,16 @@ -import { useState, useEffect, useRef, useEffectEvent } from 'react' -import { subjectsApi } from '../lib/api/subjects'; -import { studiesApi } from '../lib/api/studies'; +import { useState, useEffect, useRef, type ReactNode } from 'react' import type { StudySubject } from '../lib/api/types'; import { useClickOutside } from '../hooks/useClickOutside' +import { useDebouncedValue } from '../hooks/useDebouncedValue' import { useMergeSubjects } from '../hooks/useSubjects' -import { Modal } from '../ui' +import { useStudySubjects } from '../hooks/useStudies' +import { Modal, getQueryErrorMessage } from '../ui' import { parseDisplayDate } from '../lib/date-utils' +/** How many matches a picker lists; typing narrows the search on the server. */ +const SUBJECT_SEARCH_LIMIT = 50 +const SEARCH_DEBOUNCE_MS = 250 + interface SubjectMergeModalProps { isOpen: boolean onClose: () => void @@ -16,108 +20,199 @@ interface SubjectMergeModalProps { openKey?: number } -function SubjectMergeModalContent({ +function specimenLabel(count: number): string { + return `${count} specimen${count !== 1 ? 's' : ''}` +} + +/** + * Search box over a study's subjects. Each keystroke (debounced) searches the API, so every + * subject in the study can be found, not only one loaded page. + */ +function SubjectPicker({ + inputId, + label, + tone, studyId, - onClose, - onSuccess, + excludeId, + selected, + onSelect, }: { + inputId: string + label: ReactNode + tone: 'source' | 'target' studyId: number - onClose: () => void - onSuccess: () => void + excludeId: number | null + selected: StudySubject | null + onSelect: (subject: StudySubject | null) => void }) { - const [subjects, setSubjects] = useState([]) - const [sourceSubjectId, setSourceSubjectId] = useState(null) - const [targetSubjectId, setTargetSubjectId] = useState(null) - const [loading, setLoading] = useState(false) - const [merging, setMerging] = useState(false) - const [error, setError] = useState(null) - const [loadingCounts, setLoadingCounts] = useState(false) - const [sourceSearch, setSourceSearch] = useState('') - const [targetSearch, setTargetSearch] = useState('') - const [sourceOpen, setSourceOpen] = useState(false) - const [targetOpen, setTargetOpen] = useState(false) - const [sourceSelectedIndex, setSourceSelectedIndex] = useState(-1) - const [targetSelectedIndex, setTargetSelectedIndex] = useState(-1) - const [specimenCounts, setSpecimenCounts] = useState>(new Map()) - const mergeSubjects = useMergeSubjects() - const sourceRef = useRef(null) - const targetRef = useRef(null) - const sourceResultsRef = useRef(null) - const targetResultsRef = useRef(null) + const [search, setSearch] = useState('') + const [open, setOpen] = useState(false) + const [selectedIndex, setSelectedIndex] = useState(-1) + const containerRef = useRef(null) + const resultsRef = useRef(null) + const debouncedSearch = useDebouncedValue(search.trim(), SEARCH_DEBOUNCE_MS) - // Derived from specimenCounts map during render - const sourceSpecimenCount = sourceSubjectId ? (specimenCounts.get(sourceSubjectId) ?? null) : null - const targetSpecimenCount = targetSubjectId ? (specimenCounts.get(targetSubjectId) ?? null) : null + const subjectsQuery = useStudySubjects( + studyId, + { page: 1, limit: SUBJECT_SEARCH_LIMIT, ...(debouncedSearch ? { search: debouncedSearch } : {}) }, + { enabled: open }, + ) + const results = (subjectsQuery.data?.subjects ?? []).filter((s) => s.id !== excludeId) + const total = subjectsQuery.data?.pagination?.total ?? results.length - // Load subjects when content mounts (key resets state on each open) - useEffect(() => { - if (studyId) { - loadSubjects() - } - }, [studyId]) + useClickOutside(containerRef, () => setOpen(false), open) - // Scroll selected item into view useEffect(() => { - const el = sourceSelectedIndex >= 0 ? sourceResultsRef.current?.querySelector(`[data-index="${sourceSelectedIndex}"]`) as HTMLElement | null : null + const el = selectedIndex >= 0 ? resultsRef.current?.querySelector(`[data-index="${selectedIndex}"]`) as HTMLElement | null : null if (el) el.scrollIntoView({ block: 'nearest', behavior: 'smooth' }) - }, [sourceSelectedIndex]) + }, [selectedIndex]) - useEffect(() => { - const el = targetSelectedIndex >= 0 ? targetResultsRef.current?.querySelector(`[data-index="${targetSelectedIndex}"]`) as HTMLElement | null : null - if (el) el.scrollIntoView({ block: 'nearest', behavior: 'smooth' }) - }, [targetSelectedIndex]) + const choose = (subject: StudySubject) => { + onSelect(subject) + setSearch(subject.name) + setOpen(false) + } - useClickOutside(sourceRef, () => setSourceOpen(false), sourceOpen) - useClickOutside(targetRef, () => setTargetOpen(false), targetOpen) + const dropdownStyle = { + position: 'fixed' as const, + width: containerRef.current?.offsetWidth + 'px' || 'auto', + top: containerRef.current ? (containerRef.current.getBoundingClientRect().bottom + window.scrollY + 4) + 'px' : 'auto', + left: containerRef.current ? (containerRef.current.getBoundingClientRect().left + window.scrollX) + 'px' : 'auto', + } + const panelClass = 'absolute z-[200] w-full mt-1 bg-app-card border border-app-border rounded-md shadow-lg' + const messageClass = `${panelClass} p-4 text-sm text-app-text-muted` + const focusRing = tone === 'source' ? 'focus:ring-app-trend-down focus:border-app-trend-down' : 'focus:ring-app-accent focus:border-app-accent' - const loadAllSpecimenCounts = async (subjectsList: StudySubject[]) => { - if (subjectsList.length === 0) return - try { - const counts = new Map() - const toLoad = subjectsList.slice(0, 50) - const promises = toLoad.map(async (subject) => { - try { - const summary = await subjectsApi.getSummary(subject.id) - counts.set(subject.id, summary.summary.totalSpecimens) - } catch (err) { - console.error(`Failed to load count for subject ${subject.id}:`, err) - counts.set(subject.id, 0) - } - }) - await Promise.all(promises) - setSpecimenCounts(counts) - } catch (err) { - console.error('Failed to load specimen counts:', err) + let dropdown: ReactNode = null + if (open) { + if (subjectsQuery.isError) { + dropdown = ( +
+ {getQueryErrorMessage(subjectsQuery.error, 'Failed to load subjects')} +
+ ) + } else if (subjectsQuery.isPending) { + dropdown =
Searching subjects...
+ } else if (results.length === 0) { + dropdown = ( +
+ {debouncedSearch ? `No subjects found matching "${debouncedSearch}"` : 'No subjects found in this study'} +
+ ) + } else { + dropdown = ( +
+
    + {results.map((subject, index) => ( +
  • + +
    • + ))} +
+ {total > results.length && ( +

+ Showing {results.length} of {total.toLocaleString('en-US')}. Type to narrow the search. +

+ )} +
+ ) } } - const loadSubjects = useEffectEvent(async () => { - if (!studyId) { - setError('Study ID is required') - setLoading(false) - return - } + return ( + <> + +
+
+ { + setSearch(e.target.value) + setOpen(true) + setSelectedIndex(0) + if (selected) onSelect(null) + }} + onFocus={() => setOpen(true)} + onKeyDown={(e) => { + if (e.key === 'ArrowDown') { + e.preventDefault() + setSelectedIndex(prev => (prev < results.length - 1 ? prev + 1 : prev)) + } else if (e.key === 'ArrowUp') { + e.preventDefault() + setSelectedIndex(prev => (prev > 0 ? prev - 1 : 0)) + } else if (e.key === 'Enter' && selectedIndex >= 0 && results[selectedIndex]) { + e.preventDefault() + choose(results[selectedIndex]) + } else if (e.key === 'Escape') { + setOpen(false) + } + }} + placeholder="Search by name or ID..." + autoComplete="off" + className={`w-full px-3 py-2 pr-8 border border-app-border rounded-md shadow-sm focus:outline-none ${focusRing} bg-app-card`} + /> + + + +
+ {dropdown} +
+ {selected && ( +
+
+ Specimens: {selected.specimenCount ?? '...'} +
+
+ Created: {parseDisplayDate(selected.created).toLocaleDateString()} +
+
+ )} + + ) +} - try { - setLoading(true) - setError(null) - const response = await studiesApi.getSubjects(studyId, { page: 1, limit: 1000 }) - const subjectsList = response.subjects - setSubjects(subjectsList) - if (subjectsList.length === 0) { - setError('No subjects found in this study') - } else { - void loadAllSpecimenCounts(subjectsList) - } - } catch (err: unknown) { - console.error('Failed to load subjects:', err) - const errObj = err as { response?: { data?: { error?: string } } } - setError(errObj.response?.data?.error || 'Failed to load subjects') - setSubjects([]) - } finally { - setLoading(false) - } - }) +function SubjectMergeModalContent({ + studyId, + onClose, + onSuccess, +}: { + studyId: number + onClose: () => void + onSuccess: () => void +}) { + const [sourceSubject, setSourceSubject] = useState(null) + const [targetSubject, setTargetSubject] = useState(null) + const [merging, setMerging] = useState(false) + const [error, setError] = useState(null) + const mergeSubjects = useMergeSubjects() + const sourceSubjectId = sourceSubject?.id ?? null + const targetSubjectId = targetSubject?.id ?? null + const sourceSpecimenCount = sourceSubject?.specimenCount ?? null + const targetSpecimenCount = targetSubject?.specimenCount ?? null const handleMerge = async () => { if (!sourceSubjectId || !targetSubjectId) { @@ -154,28 +249,6 @@ function SubjectMergeModalContent({ } } - const sourceSubject = subjects.find(s => s.id === sourceSubjectId) - const targetSubject = subjects.find(s => s.id === targetSubjectId) - - // Filter subjects for dropdowns - exclude the other selected subject and filter by search - const availableSourceSubjects = subjects - .filter(s => s.id !== targetSubjectId) - .filter(s => { - if (!sourceSearch) return true - const searchLower = sourceSearch.toLowerCase() - return s.name.toLowerCase().includes(searchLower) || - s.id.toString().includes(searchLower) - }) - - const availableTargetSubjects = subjects - .filter(s => s.id !== sourceSubjectId) - .filter(s => { - if (!targetSearch) return true - const searchLower = targetSearch.toLowerCase() - return s.name.toLowerCase().includes(searchLower) || - s.id.toString().includes(searchLower) - }) - return ( <>
@@ -199,256 +272,28 @@ function SubjectMergeModalContent({
{/* Source Subject Selection */}
- - {loading ? ( -
Loading subjects...
- ) : ( -
-
- { - setSourceSearch(e.target.value) - setSourceOpen(true) - setSourceSelectedIndex(0) - if (!e.target.value) { - setSourceSubjectId(null) - } - }} - onFocus={() => setSourceOpen(true)} - onKeyDown={(e) => { - if (e.key === 'ArrowDown') { - e.preventDefault() - setSourceSelectedIndex(prev => - prev < availableSourceSubjects.length - 1 ? prev + 1 : prev - ) - } else if (e.key === 'ArrowUp') { - e.preventDefault() - setSourceSelectedIndex(prev => prev > 0 ? prev - 1 : 0) - } else if (e.key === 'Enter' && sourceSelectedIndex >= 0) { - e.preventDefault() - const selected = availableSourceSubjects[sourceSelectedIndex] - setSourceSubjectId(selected.id) - setSourceSearch(selected.name) - setSourceOpen(false) - } else if (e.key === 'Escape') { - setSourceOpen(false) - } - }} - placeholder="Search by name or ID..." - className="w-full px-3 py-2 pr-8 border border-app-border rounded-md shadow-sm focus:outline-none focus:ring-app-trend-down focus:border-app-trend-down bg-app-card" - /> - - - -
- {sourceOpen && availableSourceSubjects.length > 0 && ( -
-
    - {availableSourceSubjects.map((subject, index) => { - const count = specimenCounts.get(subject.id) ?? null - return ( -
  • - -
    • - ) - })} -
-
- )} - {sourceOpen && sourceSearch && availableSourceSubjects.length === 0 && ( -
- No subjects found matching "{sourceSearch}" -
- )} -
- )} - {sourceSubject && ( -
-
- Specimens: {sourceSpecimenCount ?? '...'} -
-
- Created: {parseDisplayDate(sourceSubject.created).toLocaleDateString()} -
-
- )} + Merge FROM (will be deleted)} + tone="source" + studyId={studyId} + excludeId={targetSubjectId} + selected={sourceSubject} + onSelect={setSourceSubject} + />
{/* Target Subject Selection */}
- - {loading ? ( -
Loading subjects...
- ) : ( -
-
- { - setTargetSearch(e.target.value) - setTargetOpen(true) - setTargetSelectedIndex(0) - if (!e.target.value) { - setTargetSubjectId(null) - } - }} - onFocus={() => setTargetOpen(true)} - onKeyDown={(e) => { - if (e.key === 'ArrowDown') { - e.preventDefault() - setTargetSelectedIndex(prev => - prev < availableTargetSubjects.length - 1 ? prev + 1 : prev - ) - } else if (e.key === 'ArrowUp') { - e.preventDefault() - setTargetSelectedIndex(prev => prev > 0 ? prev - 1 : 0) - } else if (e.key === 'Enter' && targetSelectedIndex >= 0) { - e.preventDefault() - const selected = availableTargetSubjects[targetSelectedIndex] - setTargetSubjectId(selected.id) - setTargetSearch(selected.name) - setTargetOpen(false) - } else if (e.key === 'Escape') { - setTargetOpen(false) - } - }} - placeholder="Search by name or ID..." - className="w-full px-3 py-2 pr-8 border border-app-border rounded-md shadow-sm focus:outline-none focus:ring-app-accent focus:border-app-accent bg-app-card" - /> - - - -
- {targetOpen && availableTargetSubjects.length > 0 && ( -
-
    - {availableTargetSubjects.map((subject, index) => { - const count = specimenCounts.get(subject.id) ?? null - return ( -
  • - -
    • - ) - })} -
-
- )} - {targetOpen && targetSearch && availableTargetSubjects.length === 0 && ( -
- No subjects found matching "{targetSearch}" -
- )} -
- )} - {targetSubject && ( -
-
- Specimens: {targetSpecimenCount ?? '...'} -
-
- Created: {parseDisplayDate(targetSubject.created).toLocaleDateString()} -
-
- )} + Merge INTO (will be kept)} + tone="target" + studyId={studyId} + excludeId={sourceSubjectId} + selected={targetSubject} + onSelect={setTargetSubject} + />
diff --git a/packages/web/src/components/__tests__/SubjectMergeModal.test.tsx b/packages/web/src/components/__tests__/SubjectMergeModal.test.tsx new file mode 100644 index 00000000..54179043 --- /dev/null +++ b/packages/web/src/components/__tests__/SubjectMergeModal.test.tsx @@ -0,0 +1,128 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import userEvent from '@testing-library/user-event' +import { render, screen, waitFor } from '../../__tests__/helpers/render' +import SubjectMergeModal from '../SubjectMergeModal' +import type { StudySubject } from '../../lib/api/types' + +vi.mock('../../lib/api/studies', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('studies', { + studiesApi: { getSubjects: vi.fn() }, + }) +}) + +vi.mock('../../lib/api/subjects', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('subjects', { + subjectsApi: { merge: vi.fn() }, + }) +}) + +import { studiesApi } from '../../lib/api/studies' +import { subjectsApi } from '../../lib/api/subjects' + +const STUDY_ID = 7 + +function subject(id: number): StudySubject { + return { + id, + studyId: STUDY_ID, + name: `P${String(id).padStart(5, '0')}`, + created: '2024-01-01 00:00:00', + lastUpdated: '2024-01-01 00:00:00', + specimenCount: id % 3, + } +} + +/** 1,100 subjects in the study; the mock searches and pages them as the API does. */ +const allSubjects = Array.from({ length: 1_100 }, (_, i) => subject(i + 1)) + +function serveSubjects() { + vi.mocked(studiesApi.getSubjects).mockImplementation(async (_id, params) => { + const search = params?.search?.toLowerCase() ?? '' + const matches = allSubjects.filter((s) => !search || s.name.toLowerCase().includes(search) || String(s.id) === search) + const limit = params?.limit ?? 50 + return { + subjects: matches.slice(0, limit), + pagination: { page: 1, limit, total: matches.length, totalPages: Math.ceil(matches.length / limit) }, + } + }) +} + +async function renderModal() { + await render() +} + +describe('SubjectMergeModal', () => { + beforeEach(() => { + vi.mocked(studiesApi.getSubjects).mockReset() + vi.mocked(subjectsApi.merge).mockReset() + vi.spyOn(window, 'alert').mockImplementation(() => {}) + }) + + it('searches the API, so a subject past the first 1,000 can be merged', async () => { + serveSubjects() + vi.mocked(subjectsApi.merge).mockResolvedValue({ + success: true, + specimensTransferred: 1, + specimensMerged: 0, + containersMerged: 0, + totalContainersTransferred: 1, + targetSubject: subject(2), + }) + const user = userEvent.setup() + await renderModal() + + await user.type(screen.getByLabelText(/merge from/i), 'P01050') + await user.click(await screen.findByRole('button', { name: /P01050/ })) + + await user.type(screen.getByLabelText(/merge into/i), 'P00002') + await user.click(await screen.findByRole('button', { name: /P00002/ })) + + await user.click(screen.getByRole('button', { name: 'Merge Subjects' })) + + await waitFor(() => expect(subjectsApi.merge).toHaveBeenCalledWith(2, 1050)) + expect(studiesApi.getSubjects).toHaveBeenCalledWith(STUDY_ID, expect.objectContaining({ search: 'P01050' })) + }) + + it('shows the specimen count the search returns', async () => { + serveSubjects() + const user = userEvent.setup() + await renderModal() + + await user.type(screen.getByLabelText(/merge from/i), 'P01052') + const option = await screen.findByRole('button', { name: /P01052/ }) + + expect(option).toHaveTextContent('2 specimens') + }) + + it('says when more subjects match than it lists', async () => { + serveSubjects() + const user = userEvent.setup() + await renderModal() + + await user.click(screen.getByLabelText(/merge from/i)) + + expect(await screen.findByText(/showing 50 of 1,100/i)).toBeInTheDocument() + }) + + it('shows a loading state while the search runs', async () => { + vi.mocked(studiesApi.getSubjects).mockReturnValue(new Promise(() => {})) + const user = userEvent.setup() + await renderModal() + + await user.type(screen.getByLabelText(/merge from/i), 'P0') + + expect(await screen.findByText(/searching subjects/i)).toBeInTheDocument() + }) + + it('shows the API error when the search fails', async () => { + vi.mocked(studiesApi.getSubjects).mockRejectedValue({ response: { data: { error: 'Database is locked' } } }) + const user = userEvent.setup() + await renderModal() + + await user.type(screen.getByLabelText(/merge from/i), 'P0') + + expect(await screen.findByText('Database is locked')).toBeInTheDocument() + }) +}) diff --git a/packages/web/src/components/forms/SpecimenForm.tsx b/packages/web/src/components/forms/SpecimenForm.tsx index 7f26639d..dfc5a273 100644 --- a/packages/web/src/components/forms/SpecimenForm.tsx +++ b/packages/web/src/components/forms/SpecimenForm.tsx @@ -4,8 +4,11 @@ import { useNavigate } from 'react-router-dom' import StudyPicker from '../StudyPicker' import ContainerRegistration, { type ContainerData } from '../ContainerRegistration' import { useModifierHotkey } from '../../hooks/useHotkey' -import { useCreateSpecimen, type CreateSpecimenInput } from '../../hooks/useSpecimens' -import { useCreateSubject } from '../../hooks/useSubjects' +import { + useCreateSpecimen, + useCreateSpecimenWithNewSubject, + type CreateSpecimenInput, +} from '../../hooks/useSpecimens' import { useSpecimenFormCatalogs, type SpecimenFormSourceType, @@ -50,8 +53,8 @@ export default function SpecimenForm({ }: SpecimenFormProps) { const navigate = useNavigate() const createSpecimen = useCreateSpecimen({ silent: true }) - const createSubject = useCreateSubject() - const loading = createSpecimen.isPending || createSubject.isPending + const createSpecimenWithNewSubject = useCreateSpecimenWithNewSubject({ silent: true }) + const loading = createSpecimen.isPending || createSpecimenWithNewSubject.isPending const [error, setError] = useState(null) const formRef = useRef(null) const [formData, setFormData] = useState({ @@ -162,24 +165,40 @@ export default function SpecimenForm({ return } - // If creating new subject, create it first - let sourceId = formData.sourceId + const navigateToCreated = (created: Specimen) => { + if (onSuccess) { + onSuccess() + } else if (created.id) { + navigate(`/specimens/${created.id}`) + } else { + navigate('/statistics') + } + } + + // A new subject is created with its specimen in one request, so a specimen that fails + // (e.g. a duplicate barcode) leaves no subject behind and a retry starts clean. if (formData.sourceType === 'subject' && formData.createNewSubject) { - if (!formData.studyId || !formData.subjectName.trim()) { + const studyShortCode = + formData.studyShortCode || studies.find((s) => s.id === formData.studyId)?.shortCode || '' + if (!formData.studyId || !studyShortCode || !formData.subjectName.trim()) { setError('Please select a study and enter a subject name') return } - - try { - const subject = await createSubject.mutateAsync({ - studyId: formData.studyId, - name: formData.subjectName.trim(), - }) - sourceId = subject.id - } catch (err: unknown) { - setError(getApiErrorMessage(err, 'Failed to create subject')) + const specimenTypeName = + formData.specimenTypeName || specimenTypes.find((t) => t.id === formData.specimenTypeId)?.name + if (!specimenTypeName) { + setError('Please select a specimen type') return } + const created = await createSpecimenWithNewSubject.mutateAsync({ + studyShortCode, + subjectName: formData.subjectName.trim(), + specimenTypeName, + ...(formData.collectionDate ? { collectionDate: formData.collectionDate } : {}), + ...(containerData ? { container: containerData } : {}), + }) + navigateToCreated(created) + return } const data: CreateSpecimenInput = { @@ -191,8 +210,6 @@ export default function SpecimenForm({ if (formData.sourceType === 'subject') { if (subjectId) { data.sourceId = subjectId - } else if (formData.createNewSubject) { - data.sourceId = sourceId } else if (formData.sourceId) { data.sourceId = formData.sourceId } else if (formData.studyShortCode && formData.subjectName) { @@ -229,13 +246,7 @@ export default function SpecimenForm({ } const created = await createSpecimen.mutateAsync(data) - if (onSuccess) { - onSuccess() - } else if (created.id) { - navigate(`/specimens/${created.id}`) - } else { - navigate('/statistics') - } + navigateToCreated(created) } catch (err: unknown) { setError(getApiErrorMessage(err, 'Failed to save specimen')) } @@ -265,7 +276,7 @@ export default function SpecimenForm({ )} {error && ( -
+
{error}
)} diff --git a/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx b/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx index 3b7aef6d..829f04c2 100644 --- a/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx +++ b/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx @@ -1,6 +1,9 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' -import { render, screen } from '../../../__tests__/helpers/render' +import userEvent from '@testing-library/user-event' +import { render, screen, waitFor } from '../../../__tests__/helpers/render' import SpecimenForm from '../SpecimenForm' +import { subjectsApi } from '../../../lib/api/subjects' +import { specimensApi } from '../../../lib/api/specimens' vi.mock('../../../lib/api/studies', async () => { const { createMockedDomainModule } = await import('../../../__tests__/helpers/mock-api') @@ -71,4 +74,62 @@ describe('SpecimenForm', () => { expect(screen.getByLabelText(/source type \*/i)).toBeInTheDocument() expect(screen.getByLabelText(/specimen type \*/i)).toBeInTheDocument() }) + + describe('Create New Subject', () => { + const created = { + subject: { id: 9, studyId: 1, name: 'NEW-1', created: '2024-01-01', lastUpdated: '2024-01-01' }, + subjectCreated: true, + specimens: [ + { id: 4, studySubjectId: 9, specimenTypeId: 1, created: '2024-01-01', lastUpdated: '2024-01-01', containerCreated: false }, + ], + summary: { subjectsCreated: 1, subjectsUpdated: 0, specimensCreated: 1, containersCreated: 0 }, + } + + async function fillNewSubject(user: ReturnType) { + await user.click(await screen.findByRole('checkbox', { name: /create new subject/i })) + await user.type(screen.getByLabelText(/subject name \*/i), 'NEW-1') + await screen.findByRole('option', { name: 'Blood' }) + await user.selectOptions(screen.getByLabelText(/specimen type \*/i), 'Blood') + } + + it('creates the subject and the specimen in one request', async () => { + vi.mocked(subjectsApi.createWithSpecimens).mockResolvedValue(created) + const onSuccess = vi.fn() + const user = userEvent.setup() + await render() + + await fillNewSubject(user) + await user.click(screen.getByRole('button', { name: 'Create' })) + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(subjectsApi.createWithSpecimens).toHaveBeenCalledWith({ + studyShortCode: 'SA', + subjectName: 'NEW-1', + specimens: [{ specimenTypeName: 'Blood' }], + }) + expect(subjectsApi.create).not.toHaveBeenCalled() + expect(specimensApi.create).not.toHaveBeenCalled() + }) + + it('says why the specimen failed and lets the retry create the subject', async () => { + vi.mocked(subjectsApi.createWithSpecimens) + .mockRejectedValueOnce({ response: { status: 409, data: { error: "Barcode 'X1' already exists" } } }) + .mockResolvedValueOnce(created) + const onSuccess = vi.fn() + const user = userEvent.setup() + await render() + + await fillNewSubject(user) + await user.click(screen.getByRole('button', { name: 'Create' })) + + expect(await screen.findByRole('alert')).toHaveTextContent("Barcode 'X1' already exists") + expect(onSuccess).not.toHaveBeenCalled() + expect(subjectsApi.create).not.toHaveBeenCalled() + + await user.click(screen.getByRole('button', { name: 'Create' })) + + await waitFor(() => expect(onSuccess).toHaveBeenCalled()) + expect(subjectsApi.createWithSpecimens).toHaveBeenCalledTimes(2) + }) + }) }) diff --git a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx index f627ba43..4adc80d1 100644 --- a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx +++ b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx @@ -7,11 +7,14 @@ import { useSpecimens, useSpecimen, useCreateSpecimen, + useCreateSpecimenWithNewSubject, specimenKeys, } from '../useSpecimens' import { dashboardKeys } from '../useDashboard' import { studyKeys } from '../useStudies' import { specimensApi } from '../../lib/api/specimens' +import { subjectsApi } from '../../lib/api/subjects' +import { subjectKeys } from '../useSubjects' // Mock the API module vi.mock('../../lib/api/specimens', async () => { @@ -20,6 +23,13 @@ vi.mock('../../lib/api/specimens', async () => { return createMockedDomainModule('specimens', specimensHooksMock()) }) +vi.mock('../../lib/api/subjects', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('subjects', { + subjectsApi: { createWithSpecimens: vi.fn() }, + }) +}) + function createWrapper() { const queryClient = new QueryClient({ defaultOptions: { @@ -194,7 +204,55 @@ describe('useSpecimens Hooks', () => { }) }) }) -}) + describe('useCreateSpecimenWithNewSubject', () => { + it('creates the subject and specimen in one request and invalidates their queries', async () => { + vi.mocked(subjectsApi.createWithSpecimens).mockResolvedValue({ + subject: { id: 10, studyId: 5, name: 'NEW-1', created: '2024-01-01', lastUpdated: '2024-01-01' }, + subjectCreated: true, + specimens: [ + { + id: 3, + studySubjectId: 10, + specimenTypeId: 1, + created: '2024-01-01', + lastUpdated: '2024-01-01', + containerCreated: false, + }, + ], + summary: { subjectsCreated: 1, subjectsUpdated: 0, specimensCreated: 1, containersCreated: 0 }, + }) + const queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false, gcTime: 0 }, mutations: { retry: false } }, + }) + const wrapper = ({ children }: { children: ReactNode }) => ( + + {children} + + ) + const { result } = renderHook(() => useCreateSpecimenWithNewSubject(), { wrapper }) + const invalidateSpy = vi.spyOn(queryClient, 'invalidateQueries') + result.current.mutate({ + studyShortCode: 'ST5', + subjectName: 'NEW-1', + specimenTypeName: 'Blood', + collectionDate: '2024-02-03', + }) + await waitFor(() => expect(result.current.isSuccess).toBe(true)) + expect(subjectsApi.createWithSpecimens).toHaveBeenCalledWith({ + studyShortCode: 'ST5', + subjectName: 'NEW-1', + specimens: [{ specimenTypeName: 'Blood', collectionDate: '2024-02-03' }], + }) + expect(result.current.data).toMatchObject({ id: 3, studySubjectId: 10 }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: specimenKeys.lists() }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: dashboardKeys.all }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: subjectKeys.detail(10) }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: subjectKeys.summary(10) }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: [...studyKeys.detail(5), 'subjects'] }) + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: [...studyKeys.detail(5), 'summary'] }) + }) + }) +}) diff --git a/packages/web/src/hooks/useSpecimens.ts b/packages/web/src/hooks/useSpecimens.ts index 4d542b7f..477a5f83 100644 --- a/packages/web/src/hooks/useSpecimens.ts +++ b/packages/web/src/hooks/useSpecimens.ts @@ -97,6 +97,56 @@ export function invalidateSpecimenQueries( } } +export type CreateSpecimenWithNewSubjectInput = { + studyShortCode: string + subjectName: string + specimenTypeName: string + collectionDate?: string + /** Flat form state; mapped to write shape before API call. */ + container?: ContainerData | SpecimenContainerWriteInput +} + +/** + * Create a subject and its first specimen in one request (`POST /subjects/with-specimens`). + * The API writes both in one transaction, so a specimen that fails leaves no subject behind. + * A subject already named so in the study is reused. + */ +export function useCreateSpecimenWithNewSubject(options?: { silent?: boolean }) { + const queryClient = useQueryClient() + const { success, error: showError } = useToast() + + return useMutation({ + mutationFn: async ({ studyShortCode, subjectName, container, ...specimen }: CreateSpecimenWithNewSubjectInput) => { + const res = await subjectsApi.createWithSpecimens({ + studyShortCode, + subjectName, + specimens: [ + { + ...specimen, + ...(container + ? { + container: isFlatContainerRegistration(container) + ? flatContainerRegistrationToWriteInput(container) + : container, + } + : {}), + }, + ], + }) + if (res.specimens.length === 0) throw new Error('The API created no specimen') + return { ...res.specimens[0], studyId: res.subject.studyId } + }, + onError: (err: unknown) => { + if (options?.silent) return + showError(getMutationErrorMessage(err, 'Failed to create specimen')) + }, + onSuccess: (specimen) => { + invalidateSpecimenQueries(queryClient, { specimen }) + success('Specimen created successfully') + }, + }) +} + /** * One page of specimens and the pagination totals. The previous page stays on screen while * the next page, sort, or filter loads. diff --git a/packages/web/src/hooks/useStudies.ts b/packages/web/src/hooks/useStudies.ts index 9648c143..20f5e426 100644 --- a/packages/web/src/hooks/useStudies.ts +++ b/packages/web/src/hooks/useStudies.ts @@ -189,14 +189,22 @@ export function useStudy(id: number) { }) } -export function useStudySubjects(studyId: number, params?: { page?: number; limit?: number }) { +/** + * One page of a study's subjects. `search` matches a name containing the text or the exact + * subject ID, on the server. + */ +export function useStudySubjects( + studyId: number, + params?: { page?: number; limit?: number; search?: string }, + options?: { enabled?: boolean }, +) { return useQuery({ queryKey: [...studyKeys.detail(studyId), 'subjects', params], queryFn: async () => { const res = await studiesApi.getSubjects(studyId, params) return res }, - enabled: !!studyId, + enabled: !!studyId && (options?.enabled ?? true), }) } diff --git a/packages/web/src/lib/api/studies.ts b/packages/web/src/lib/api/studies.ts index ab8ca9a7..cae4c14d 100644 --- a/packages/web/src/lib/api/studies.ts +++ b/packages/web/src/lib/api/studies.ts @@ -60,7 +60,7 @@ export const studiesApi = { ) => api.get('/studies', { params: { search, ...params } }), get: (id: number) => api.get(`/studies/${id}`), - getSubjects: (id: number, params?: { page?: number; limit?: number }) => + getSubjects: (id: number, params?: { page?: number; limit?: number; search?: string }) => api.get(`/studies/${id}/subjects`, { params }), getSummary: (id: number) => api.get(`/studies/${id}/summary`), getSummaries: (ids: number[]) => diff --git a/packages/web/src/lib/api/subjects.ts b/packages/web/src/lib/api/subjects.ts index f73b24a5..de209af6 100644 --- a/packages/web/src/lib/api/subjects.ts +++ b/packages/web/src/lib/api/subjects.ts @@ -103,11 +103,7 @@ export const subjectsApi = { api.post<{ subject: StudySubject subjectCreated: boolean - specimens: Array<{ - specimen: Specimen - containerCreated: boolean - containerId?: number - }> + specimens: Array summary: { subjectsCreated: number subjectsUpdated: number From 18e3c5518e3ead399dc47926a239600ef1ef1dbc Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:23:42 -0800 Subject: [PATCH 41/54] fix(subjects): Create New Subject refuses a name the study already has (#135) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit POST /subjects/with-specimens takes `newSubjectOnly`. When set and the study already has a subject with that name, the transaction is rolled back and the route returns 409 "Subject name '…' already exists in this study", instead of adding the specimen to that subject. The specimen form sends it, so a mistyped or reused name can't silently attach a specimen to another subject. Bulk import is unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/bulk-combined-import.ts | 13 ++++-- packages/api/src/lib/schemas.ts | 2 + .../__tests__/subjects-with-specimens.test.ts | 42 +++++++++++++++++++ .../guides/workflows/subjects-specimens.md | 2 +- .../forms/__tests__/SpecimenForm.test.tsx | 1 + .../src/hooks/__tests__/useSpecimens.test.tsx | 1 + packages/web/src/hooks/useSpecimens.ts | 3 +- packages/web/src/lib/api/subjects.ts | 2 + 8 files changed, 61 insertions(+), 5 deletions(-) diff --git a/packages/api/src/lib/bulk-combined-import.ts b/packages/api/src/lib/bulk-combined-import.ts index 0476853b..48392866 100644 --- a/packages/api/src/lib/bulk-combined-import.ts +++ b/packages/api/src/lib/bulk-combined-import.ts @@ -41,6 +41,8 @@ export interface BulkCombinedSubjectInput { export interface WithSpecimensPayload { studyShortCode: string subjectName: string + /** Refuse a subject name the study already has (409) instead of adding to that subject. */ + newSubjectOnly?: boolean specimens: BulkCombinedSpecimenInput[] } @@ -229,9 +231,14 @@ export async function runOneSubjectWithSpecimens( userId: number | undefined ): Promise { try { - const [result] = await withWriteTransaction(database, (tx) => - registerSubjects(tx, payload.studyShortCode, [{ subjectName: payload.subjectName, specimens: payload.specimens }], userId) - ) + const [result] = await withWriteTransaction(database, async (tx) => { + const results = await registerSubjects(tx, payload.studyShortCode, [{ subjectName: payload.subjectName, specimens: payload.specimens }], userId) + // Throwing here rolls the transaction back, so the specimens aren't added to the existing subject. + if (payload.newSubjectOnly && !results[0].subjectCreated) { + throw new ConflictError(`Subject name '${payload.subjectName.trim()}' already exists in this study`) + } + return results + }) return result } catch (error) { if (!(error instanceof BulkCombinedRejected)) throw error diff --git a/packages/api/src/lib/schemas.ts b/packages/api/src/lib/schemas.ts index 4bef6d11..0aaad847 100644 --- a/packages/api/src/lib/schemas.ts +++ b/packages/api/src/lib/schemas.ts @@ -27,6 +27,8 @@ export const containerSchemaWithLocation = bulkCombinedContainerSchema export const withSpecimensRequestSchema = z.object({ studyShortCode: z.string().min(1), subjectName: z.string().min(1), + /** Refuse a subject name the study already has, instead of adding the specimens to that subject. */ + newSubjectOnly: z.boolean().optional(), specimens: z.array( z.object({ specimenTypeName: z.string().min(1), diff --git a/packages/api/src/routes/__tests__/subjects-with-specimens.test.ts b/packages/api/src/routes/__tests__/subjects-with-specimens.test.ts index d6d8ed1b..d6c9443f 100644 --- a/packages/api/src/routes/__tests__/subjects-with-specimens.test.ts +++ b/packages/api/src/routes/__tests__/subjects-with-specimens.test.ts @@ -549,6 +549,48 @@ describe('Subjects with Specimens API', () => { expect(data.specimens[0].containerCreated).toBe(true) }) + it('with newSubjectOnly, refuses a name the study already has and saves nothing', async () => { + const existingSubject = await createTestStudySubject(testDb, { + studyId: testStudy.id, + name: 'EXISTING-SUBJ', + }) + + const res = await authenticatedRequest(ctx.createRequestApp(), '/api/subjects/with-specimens', { + method: 'POST', + cookie: ctx.cookie, + json: { + studyShortCode: 'TEST01', + subjectName: 'EXISTING-SUBJ', + newSubjectOnly: true, + specimens: [{ specimenTypeName: 'Whole Blood' }], + }, + }) + + expect(res.status).toBe(409) + const data = await res.json() as ErrorResponse + expect(data.error).toBe("Subject name 'EXISTING-SUBJ' already exists in this study") + const saved = await testDb.select().from(specimen).where(eq(specimen.studySubjectId, existingSubject.id)) + expect(saved).toHaveLength(0) + }) + + it('with newSubjectOnly, creates a subject whose name is new', async () => { + const res = await authenticatedRequest(ctx.createRequestApp(), '/api/subjects/with-specimens', { + method: 'POST', + cookie: ctx.cookie, + json: { + studyShortCode: 'TEST01', + subjectName: 'BRAND-NEW-SUBJ', + newSubjectOnly: true, + specimens: [{ specimenTypeName: 'Whole Blood' }], + }, + }) + + expect(res.status).toBe(201) + const data = await res.json() as SubjectWithSpecimensResponse + expect(data.subjectCreated).toBe(true) + expect(data.summary.specimensCreated).toBe(1) + }) + it('should add multiple specimens to existing subject (get-or-create: one specimen, three containers)', async () => { // Create existing subject with one specimen (Whole Blood, no collection date) const existingSubject = await createTestStudySubject(testDb, { diff --git a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md index b6c28583..f24a5fa7 100644 --- a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md +++ b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md @@ -29,7 +29,7 @@ Open the form from any of these places: ### Specimen fields 1. Select a source type. **Subject** is the usual choice. The other options are **Control**, **Reagent**, **Cell Line**, **Plasmid**, and **Standard**. -2. If the source is **Subject**, select the study or enter its short code, then select the subject. To create the subject in the same step, select the **Create New Subject** checkbox and enter the name. The subject and specimen are saved together: if the specimen can't be saved, for example because a barcode is already in use, the subject isn't created either, and the error says why. If the study already has a subject with that name, the specimen is added to it. +2. If the source is **Subject**, select the study or enter its short code, then select the subject. To create the subject in the same step, select the **Create New Subject** checkbox and enter the name. The subject and specimen are saved together: if the specimen can't be saved, for example because a barcode is already in use, the subject isn't created either, and the error says why. If the study already has a subject with that name, nothing is saved and the error says so: clear the checkbox and select that subject instead. 3. Select a **Specimen Type** from Reference Data. 4. Optional: enter **Collection Date** as YYYY-MM-DD. If you leave it blank, the specimen is saved without a date. SampleDB does not default to today. diff --git a/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx b/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx index 829f04c2..d0a71e09 100644 --- a/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx +++ b/packages/web/src/components/forms/__tests__/SpecimenForm.test.tsx @@ -105,6 +105,7 @@ describe('SpecimenForm', () => { expect(subjectsApi.createWithSpecimens).toHaveBeenCalledWith({ studyShortCode: 'SA', subjectName: 'NEW-1', + newSubjectOnly: true, specimens: [{ specimenTypeName: 'Blood' }], }) expect(subjectsApi.create).not.toHaveBeenCalled() diff --git a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx index 4adc80d1..562406c0 100644 --- a/packages/web/src/hooks/__tests__/useSpecimens.test.tsx +++ b/packages/web/src/hooks/__tests__/useSpecimens.test.tsx @@ -244,6 +244,7 @@ describe('useSpecimens Hooks', () => { expect(subjectsApi.createWithSpecimens).toHaveBeenCalledWith({ studyShortCode: 'ST5', subjectName: 'NEW-1', + newSubjectOnly: true, specimens: [{ specimenTypeName: 'Blood', collectionDate: '2024-02-03' }], }) expect(result.current.data).toMatchObject({ id: 3, studySubjectId: 10 }) diff --git a/packages/web/src/hooks/useSpecimens.ts b/packages/web/src/hooks/useSpecimens.ts index 477a5f83..05f9243e 100644 --- a/packages/web/src/hooks/useSpecimens.ts +++ b/packages/web/src/hooks/useSpecimens.ts @@ -109,7 +109,7 @@ export type CreateSpecimenWithNewSubjectInput = { /** * Create a subject and its first specimen in one request (`POST /subjects/with-specimens`). * The API writes both in one transaction, so a specimen that fails leaves no subject behind. - * A subject already named so in the study is reused. + * It sends `newSubjectOnly`, so a name the study already has is refused (409) rather than reused. */ export function useCreateSpecimenWithNewSubject(options?: { silent?: boolean }) { const queryClient = useQueryClient() @@ -120,6 +120,7 @@ export function useCreateSpecimenWithNewSubject(options?: { silent?: boolean }) const res = await subjectsApi.createWithSpecimens({ studyShortCode, subjectName, + newSubjectOnly: true, specimens: [ { ...specimen, diff --git a/packages/web/src/lib/api/subjects.ts b/packages/web/src/lib/api/subjects.ts index de209af6..028eaaa2 100644 --- a/packages/web/src/lib/api/subjects.ts +++ b/packages/web/src/lib/api/subjects.ts @@ -94,6 +94,8 @@ export const subjectsApi = { createWithSpecimens: (data: { studyShortCode: string subjectName: string + /** Refuse a name the study already has (409) instead of adding to that subject. */ + newSubjectOnly?: boolean specimens: Array<{ specimenTypeName: string collectionDate?: string From 8626013719aca7b5e6d2945bc2e8112042c5f8c6 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:58:00 -0800 Subject: [PATCH 42/54] fix(statistics): report study dates and location counts that match the data (#107) - Study summaries, the study summary route and the study timeline order the collection date range by calendar date, reading legacy M/D/YYYY values as dates, and report it as YYYY-MM-DD. Study duration uses the same dates. - The study enrollment timeline buckets Subjects by UTC day of `created` instead of by the full timestamp, and both daily timelines count with a Map instead of a quadratic find. - The statistics collection timeline reads a collection date's month from the date itself, so `5/1/2024` is May in every server time zone. - Statistics `storage.byLocation` lists every Location holding Containers, grouped by Location id, so the Storage Locations card (its length) is the real count instead of at most 20. The wire shape is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../lib/__tests__/timestamp-bounds.test.ts | 14 +++ .../__tests__/storage-aggregates.test.ts | 50 +++++++++++ .../src/lib/statistics/specimen-aggregates.ts | 9 +- .../src/lib/statistics/storage-aggregates.ts | 18 ++-- .../lib/studies/__tests__/study-read.test.ts | 21 +++++ packages/api/src/lib/studies/study-dates.ts | 60 +++++++++++++ packages/api/src/lib/studies/study-read.ts | 13 +-- .../api/src/routes/__tests__/studies.test.ts | 63 ++++++++++++++ packages/api/src/routes/studies.ts | 85 ++----------------- 9 files changed, 232 insertions(+), 101 deletions(-) create mode 100644 packages/api/src/lib/studies/study-dates.ts diff --git a/packages/api/src/lib/__tests__/timestamp-bounds.test.ts b/packages/api/src/lib/__tests__/timestamp-bounds.test.ts index d960cd4e..aa9d43ac 100644 --- a/packages/api/src/lib/__tests__/timestamp-bounds.test.ts +++ b/packages/api/src/lib/__tests__/timestamp-bounds.test.ts @@ -107,6 +107,20 @@ describe('timestamp bounds against stored timestamps', () => { { date: '2024-02', count: 1 }, ]) }) + + it('buckets a collection date written M/D/YYYY by its own month east of UTC (#107)', async () => { + process.env.TZ = 'Pacific/Auckland' + try { + const type = await createTestSpecimenType(db, { name: 'Blood' }) + const legacy = await createTestSpecimen(db, type.id, { collectionDate: '5/1/2024' }) + + const result = await computeSpecimenAggregates(db, [legacy], {}) + + expect(result.collectionTimeline).toEqual([{ date: '2024-05', count: 1 }]) + } finally { + process.env.TZ = 'America/Los_Angeles' + } + }) }) describe('created_from / created_to filters', () => { diff --git a/packages/api/src/lib/statistics/__tests__/storage-aggregates.test.ts b/packages/api/src/lib/statistics/__tests__/storage-aggregates.test.ts index d27a0296..f7477880 100644 --- a/packages/api/src/lib/statistics/__tests__/storage-aggregates.test.ts +++ b/packages/api/src/lib/statistics/__tests__/storage-aggregates.test.ts @@ -4,8 +4,11 @@ import { setupTestDatabase, cleanupTestDatabase } from '../../../__tests__/helpe import { createTestLocation, createTestMicronixPlate, + createTestSpecimen, + createTestSpecimenType, createTestStorageContainer, createTestStorageType, + createTestUnit, } from '../../../__tests__/helpers/factories' import { micronixTube } from '../../../db/schema' import { computeStorageStatistics } from '../storage-aggregates' @@ -63,4 +66,51 @@ describe('storage-aggregates', () => { containersWithoutLocations: 0, }) }) + + describe('locations in use (#107)', () => { + let storageTypeId: string + let unitId: number + let specimenTypeId: number + + beforeEach(async () => { + storageTypeId = String((await createTestStorageType(testDb, { name: 'Freezer' })).id) + unitId = (await createTestUnit(testDb, { symbol: 'uL', name: 'microliter', category: 'volume' })).id + specimenTypeId = (await createTestSpecimenType(testDb, { name: 'Blood' })).id + }) + + /** A micronix tube on its own plate at a new Location with this path. */ + async function tubeAtNewLocation(path: string, index: number): Promise { + const loc = await createTestLocation(testDb, { name: `${path} ${index}`, path, storageTypeId }) + const plate = await createTestMicronixPlate(testDb, { name: `Plate ${index}`, locationId: loc.id }) + const spec = await createTestSpecimen(testDb, specimenTypeId) + const container = await createTestStorageContainer(testDb, { specimenId: spec.id, unitId }) + await testDb.insert(micronixTube).values({ + id: container.id, + collectionId: plate.id, + barcode: `MX-LOC-${index}`, + position: 'A01', + }) + return container.id + } + + it('lists every Location holding Containers, not a top-20 list', async () => { + const containerIds: number[] = [] + for (let i = 0; i < 25; i++) containerIds.push(await tubeAtNewLocation(`Shelf ${i}`, i)) + + const result = await computeStorageStatistics(testDb, containerIds, containerIds.length) + + expect(result.byLocation).toHaveLength(25) + }) + + it('counts two Locations that share a path as two Locations', async () => { + const containerIds = [await tubeAtNewLocation('Shelf', 1), await tubeAtNewLocation('Shelf', 2)] + + const result = await computeStorageStatistics(testDb, containerIds, containerIds.length) + + expect(result.byLocation).toEqual([ + { location: 'Shelf', count: 1 }, + { location: 'Shelf', count: 1 }, + ]) + }) + }) }) diff --git a/packages/api/src/lib/statistics/specimen-aggregates.ts b/packages/api/src/lib/statistics/specimen-aggregates.ts index 7407bfba..2c874f93 100644 --- a/packages/api/src/lib/statistics/specimen-aggregates.ts +++ b/packages/api/src/lib/statistics/specimen-aggregates.ts @@ -4,6 +4,7 @@ import { inArray } from 'drizzle-orm' import { cache, cacheKeys } from '../cache' import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' import { utcMonthOf } from '../datetime' +import { parseCollectionDate } from '../validation' export type SpecimenAggregates = { total: number @@ -28,9 +29,11 @@ function buildMonthlyTimeline( specimens .filter((s) => (dateField === 'collectionDate' ? s.collectionDate : true)) .forEach((s) => { - const rawDate = dateField === 'collectionDate' ? s.collectionDate! : s.created - // UTC month: `created` is a UTC timestamp and a collection date has no time of day. - const monthKey = utcMonthOf(rawDate) + // A collection date has no time of day, so its month is its own (`5/1/2024` is May in every + // time zone); `created` is a UTC timestamp, so its month is the UTC month. + const monthKey = dateField === 'collectionDate' + ? parseCollectionDate(s.collectionDate!)?.slice(0, 7) ?? null + : utcMonthOf(s.created) if (!monthKey) return monthMap.set(monthKey, (monthMap.get(monthKey) || 0) + 1) }) diff --git a/packages/api/src/lib/statistics/storage-aggregates.ts b/packages/api/src/lib/statistics/storage-aggregates.ts index 0e9f5dfc..d93d2678 100644 --- a/packages/api/src/lib/statistics/storage-aggregates.ts +++ b/packages/api/src/lib/statistics/storage-aggregates.ts @@ -97,20 +97,18 @@ async function loadLocationMap(database: Database, locationIds: number[]) { return current } - const locationCountMap = new Map() + // Every Location holding Containers, busiest first. The list is not capped: its length is the + // dashboard's count of Locations in use, and the web shows only the top entries in its chart. + const containerCountByLocation = new Map() locationIds.forEach((id) => { - const loc = locationMap.get(id) - if (loc) { - const path = loc.path || loc.name || `Location ${loc.id}` - locationCountMap.set(path, (locationCountMap.get(path) || 0) + 1) - } + if (locationMap.has(id)) containerCountByLocation.set(id, (containerCountByLocation.get(id) || 0) + 1) }) - Array.from(locationCountMap.entries()) + Array.from(containerCountByLocation.entries()) .sort((a, b) => b[1] - a[1]) - .slice(0, 20) - .forEach(([locationPath, count]) => { - byLocation.push({ location: locationPath, count }) + .forEach(([id, count]) => { + const loc = locationMap.get(id)! + byLocation.push({ location: loc.path || loc.name || `Location ${loc.id}`, count }) }) locationIds.forEach((id) => { diff --git a/packages/api/src/lib/studies/__tests__/study-read.test.ts b/packages/api/src/lib/studies/__tests__/study-read.test.ts index 60454642..49bed967 100644 --- a/packages/api/src/lib/studies/__tests__/study-read.test.ts +++ b/packages/api/src/lib/studies/__tests__/study-read.test.ts @@ -140,6 +140,27 @@ describe('study-read', () => { ]) }) + it('orders the collection date range by date across mixed years and formats (#107)', async () => { + const study = await createTestStudy(testDb, { title: 'Mixed Study', shortCode: 'MIX01' }) + const subject = await createTestStudySubject(testDb, { studyId: study.id, name: 'Subj 1' }) + const specimenType = await createTestSpecimenType(testDb, { name: 'Blood' }) + const now = utcNow() + // As text, '12/1/2024' sorts first and '2023-06-01' last. + for (const collectionDate of ['12/1/2024', '2/1/2023', '2023-06-01', '2024-03-15']) { + await testDb.insert(specimen).values({ + studySubjectId: subject.id, + specimenTypeId: specimenType.id, + collectionDate, + created: now, + lastUpdated: now, + }) + } + + const result = await getStudySummaries(testDb, sqlite, [study.id]) + + expect(result.summaries[0]!.collectionDateRange).toEqual({ earliest: '2023-02-01', latest: '2024-12-01' }) + }) + it('returns empty array when ids is empty', async () => { const result = await getStudySummaries(testDb, sqlite, []) expect(result.summaries).toEqual([]) diff --git a/packages/api/src/lib/studies/study-dates.ts b/packages/api/src/lib/studies/study-dates.ts new file mode 100644 index 00000000..10b1e40d --- /dev/null +++ b/packages/api/src/lib/studies/study-dates.ts @@ -0,0 +1,60 @@ +import { parseStoredTimestamp } from '../datetime' +import { parseCollectionDate } from '../validation' + +export type DailyCount = { date: string; count: number } + +const DAY_MS = 24 * 60 * 60 * 1000 + +/** + * The calendar dates (YYYY-MM-DD) of stored collection dates. A legacy value in another format + * the collection-date rule accepts, such as `5/7/2024`, is read as that date; a value that isn't + * a date is left out. + */ +function collectionCalendarDates(dates: ReadonlyArray): string[] { + const calendarDates: string[] = [] + for (const date of dates) { + const calendarDate = date ? parseCollectionDate(date) : null + if (calendarDate) calendarDates.push(calendarDate) + } + return calendarDates +} + +/** Counts per key, in key order. */ +function countByKey(keys: Iterable): DailyCount[] { + const counts = new Map() + for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1) + return [...counts].sort(([a], [b]) => a.localeCompare(b)).map(([date, count]) => ({ date, count })) +} + +/** Earliest and latest collection dates as YYYY-MM-DD, in date order, or null when none is a date. */ +export function collectionDateRangeOf(dates: ReadonlyArray): { earliest: string; latest: string } | null { + const calendarDates = collectionCalendarDates(dates) + if (calendarDates.length === 0) return null + let earliest = calendarDates[0]! + let latest = earliest + for (const date of calendarDates) { + if (date < earliest) earliest = date + if (date > latest) latest = date + } + return { earliest, latest } +} + +/** Whole days from `earliest` to `latest`, both YYYY-MM-DD. */ +export function daysBetweenCalendarDates(earliest: string, latest: string): number { + return Math.round((Date.parse(`${latest}T00:00:00Z`) - Date.parse(`${earliest}T00:00:00Z`)) / DAY_MS) +} + +/** Specimens collected per calendar day. */ +export function collectionTimeline(dates: ReadonlyArray): DailyCount[] { + return countByKey(collectionCalendarDates(dates)) +} + +/** Subjects enrolled per UTC day of their stored `created` timestamp. */ +export function enrollmentTimeline(created: ReadonlyArray): DailyCount[] { + const days: string[] = [] + for (const value of created) { + const instant = parseStoredTimestamp(value) + if (instant) days.push(instant.toISOString().slice(0, 10)) + } + return countByKey(days) +} diff --git a/packages/api/src/lib/studies/study-read.ts b/packages/api/src/lib/studies/study-read.ts index b40cd6fc..ed015046 100644 --- a/packages/api/src/lib/studies/study-read.ts +++ b/packages/api/src/lib/studies/study-read.ts @@ -4,6 +4,7 @@ import { study, studySubject } from '../../db/schema' import { and, eq, inArray, or, sql } from 'drizzle-orm' import { SQL_IN_CHUNK_SIZE, dateRangeConditions } from '../sql-helpers' import { likeContains } from '../sql-like' +import { collectionDateRangeOf } from './study-dates' export type ListStudiesParams = { search?: string @@ -194,17 +195,7 @@ export async function getStudySummaries( const totalSpecimens = studySpecimens.length const totalContainers = containerCounts[id] || 0 - const collectionDates = (studySpecimens - .map((s) => s.collectionDate) - .filter(Boolean) as string[]) - .sort() - - const collectionDateRange = collectionDates.length > 0 - ? { - earliest: collectionDates[0], - latest: collectionDates[collectionDates.length - 1], - } - : null + const collectionDateRange = collectionDateRangeOf(studySpecimens.map((s) => s.collectionDate)) summaries.push({ studyId: id, diff --git a/packages/api/src/routes/__tests__/studies.test.ts b/packages/api/src/routes/__tests__/studies.test.ts index 58de03d7..8f789017 100644 --- a/packages/api/src/routes/__tests__/studies.test.ts +++ b/packages/api/src/routes/__tests__/studies.test.ts @@ -262,6 +262,69 @@ describe('Studies API (list, get, create, update)', () => { }) describe('GET /api/studies/:id/summary', () => { + type SummaryBody = { + summary: { + collectionDateRange: { earliest: string; latest: string } | null + studyDurationDays: number | null + collectionTimeline: Array<{ date: string; count: number }> + enrollmentTimeline: Array<{ date: string; count: number }> + } + } + + async function subjectEnrolledAt(studyId: number, name: string, created: string) { + return createTestStudySubject(ctx.db, { studyId, name, created }) + } + + it('buckets the enrollment timeline by day (#107)', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Enrollment', shortCode: 'ENR01' }) + await subjectEnrolledAt(studyRecord.id, 'A', '2024-03-01 08:00:00') + await subjectEnrolledAt(studyRecord.id, 'B', '2024-03-01 08:00:01') + await subjectEnrolledAt(studyRecord.id, 'C', '2024-03-01T23:59:59Z') + await subjectEnrolledAt(studyRecord.id, 'D', '2024-03-02 00:00:00') + + const res = await ctx.request(`/api/studies/${studyRecord.id}/summary`) + + expect(res.status).toBe(200) + const data = (await res.json()) as SummaryBody + expect(data.summary.enrollmentTimeline).toEqual([ + { date: '2024-03-01', count: 3 }, + { date: '2024-03-02', count: 1 }, + ]) + }) + + it('reports collection dates in date order across mixed years (#107)', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Mixed', shortCode: 'MIX02' }) + const subject = await subjectEnrolledAt(studyRecord.id, 'A', '2023-01-01 09:00:00') + const type = await createTestSpecimenType(ctx.db, { name: 'Blood' }) + const addSpecimen = ctx.sqlite.prepare( + "INSERT INTO specimen (study_subject_id, specimen_type_id, collection_date, created, last_updated) VALUES (?, ?, ?, '2024-01-01', '2024-01-01')", + ) + const second = await subjectEnrolledAt(studyRecord.id, 'B', '2023-01-01 17:00:00') + // As text, '12/1/2024' sorts first and '2023-06-01' last. + for (const date of ['12/1/2024', '2/1/2023', '2023-06-01']) { + addSpecimen.run(subject!.id, type.id, date) + } + addSpecimen.run(second!.id, type.id, '2023-02-01') + + const res = await ctx.request(`/api/studies/${studyRecord.id}/summary`) + + expect(res.status).toBe(200) + const data = (await res.json()) as SummaryBody + expect(data.summary.collectionDateRange).toEqual({ earliest: '2023-02-01', latest: '2024-12-01' }) + expect(data.summary.studyDurationDays).toBe(669) + expect(data.summary.collectionTimeline).toEqual([ + { date: '2023-02-01', count: 2 }, + { date: '2023-06-01', count: 1 }, + { date: '2024-12-01', count: 1 }, + ]) + expect(data.summary.enrollmentTimeline).toEqual([{ date: '2023-01-01', count: 2 }]) + + const timeline = await ctx.request(`/api/studies/${studyRecord.id}/timeline`) + expect(timeline.status).toBe(200) + const timelineData = (await timeline.json()) as { dateRange: { earliest: string; latest: string } | null } + expect(timelineData.dateRange).toEqual({ earliest: '2023-02-01', latest: '2024-12-01' }) + }) + it('handles studies with more specimens than SQLite allows as bound parameters', async () => { const studyRecord = await createTestStudy(ctx.db, { title: 'Big Study', shortCode: 'BIG01' }) const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) diff --git a/packages/api/src/routes/studies.ts b/packages/api/src/routes/studies.ts index c22a685b..aafea709 100644 --- a/packages/api/src/routes/studies.ts +++ b/packages/api/src/routes/studies.ts @@ -27,6 +27,7 @@ import { assertNotUsedInQpcr } from '../lib/qpcr-usage' import { insertedRow } from '../lib/inserted-row' import { timestampFilter } from '../lib/date-filters' import { likeContains } from '../lib/sql-like' +import { collectionDateRangeOf, collectionTimeline, daysBetweenCalendarDates, enrollmentTimeline } from '../lib/studies/study-dates' /** Short code prefix for tutorial namespace. Any study whose short code starts with this (case-insensitive) may be deleted by any member. Only admins may rename a study into or out of this namespace. */ const TUTORIAL_SHORT_CODE_PREFIX = 'TUT' @@ -301,23 +302,6 @@ studies.get('/:id/summary', authMiddleware, async (c) => { const totalSpecimens = specimens.length if (specimens.length === 0) { - // Calculate enrollment timeline from subjects - const enrollmentTimeline = subjects - .map(s => ({ - date: s.created.split('T')[0], // Just the date part - count: 1, - })) - .reduce((acc, item) => { - const existing = acc.find(x => x.date === item.date) - if (existing) { - existing.count += item.count - } else { - acc.push(item) - } - return acc - }, [] as Array<{ date: string; count: number }>) - .sort((a, b) => a.date.localeCompare(b.date)) - return c.json({ study: studyRecord, summary: { @@ -330,7 +314,7 @@ studies.get('/:id/summary', authMiddleware, async (c) => { collectionDateRange: null, studyDurationDays: null, collectionTimeline: [], - enrollmentTimeline, + enrollmentTimeline: enrollmentTimeline(subjects.map(s => s.created)), }, }) } @@ -395,58 +379,12 @@ studies.get('/:id/summary', authMiddleware, async (c) => { percentage: totalSpecimens > 0 ? (count / totalSpecimens) * 100 : 0, })) - // Collection date range - const collectionDates = (specimens - .map(s => s.collectionDate) - .filter(Boolean) as string[]) - .sort() - - const collectionDateRange = collectionDates.length > 0 - ? { - earliest: collectionDates[0], - latest: collectionDates[collectionDates.length - 1], - } - : null - - // Study duration in days + const collectionDates = specimens.map(s => s.collectionDate) + const collectionDateRange = collectionDateRangeOf(collectionDates) const studyDurationDays = collectionDateRange - ? Math.ceil( - (new Date(collectionDateRange.latest).getTime() - - new Date(collectionDateRange.earliest).getTime()) / - (1000 * 60 * 60 * 24) - ) + ? daysBetweenCalendarDates(collectionDateRange.earliest, collectionDateRange.latest) : null - // Collection timeline (daily counts) - const collectionTimeline = collectionDates.reduce((acc, date) => { - const dateOnly = date.split('T')[0] - const existing = acc.find(x => x.date === dateOnly) - if (existing) { - existing.count += 1 - } else { - acc.push({ date: dateOnly, count: 1 }) - } - return acc - }, [] as Array<{ date: string; count: number }>) - .sort((a, b) => a.date.localeCompare(b.date)) - - // Enrollment timeline (daily counts) - const enrollmentTimeline = subjects - .map(s => ({ - date: s.created.split('T')[0], - count: 1, - })) - .reduce((acc, item) => { - const existing = acc.find(x => x.date === item.date) - if (existing) { - existing.count += item.count - } else { - acc.push(item) - } - return acc - }, [] as Array<{ date: string; count: number }>) - .sort((a, b) => a.date.localeCompare(b.date)) - return c.json({ study: studyRecord, summary: { @@ -458,8 +396,8 @@ studies.get('/:id/summary', authMiddleware, async (c) => { containerTypes: containerTypeCounts, collectionDateRange, studyDurationDays, - collectionTimeline, - enrollmentTimeline, + collectionTimeline: collectionTimeline(collectionDates), + enrollmentTimeline: enrollmentTimeline(subjects.map(s => s.created)), }, }) } catch (error) { @@ -569,14 +507,7 @@ studies.get('/:id/timeline', authMiddleware, async (c) => { specimens: specimensBySubject.get(subject.id) || [], })) - // Calculate date range - const collectionDates = specimens.map(s => s.collectionDate).sort() - const dateRange = collectionDates.length > 0 - ? { - earliest: collectionDates[0], - latest: collectionDates[collectionDates.length - 1], - } - : null + const dateRange = collectionDateRangeOf(specimens.map(s => s.collectionDate)) return c.json({ subjects: subjectsWithSpecimens, From e6a5b1e75528ea0f465e0379da3a8707e0dc3c7e Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 07:59:03 -0800 Subject: [PATCH 43/54] fix(export): export modal applies Tags in CSV mode, keeps errors, and JSON rows keep every column (#122) - CSV-mode count and export send the selected Tags. - A failed export leaves the modal open with its error; only a successful manual export closes it after one second. - Each count request takes a sequence number; a response, error or loading change from a superseded request is ignored. - A JSON Container export (GET, single-study POST, multi-study, barcode) fills every export column, null when empty, and opts out of omit-on-wire with keepNullsOnWire. Other responses still omit nulls. ADR 0005 records the exception. Co-Authored-By: Claude Opus 5.5 --- docs/adr/0005-wire-dto-omit-on-wire.md | 3 +- packages/api/src/lib/export/format.ts | 15 +- .../middleware/__tests__/omit-on-wire.test.ts | 26 +++ packages/api/src/middleware/omit-on-wire.ts | 17 +- .../api/src/routes/__tests__/export.test.ts | 80 +++++++- packages/api/src/routes/export.ts | 10 + .../docs/guides/bulk-operations/export.md | 2 +- .../useSingleStudyExportWorkflow.test.tsx | 179 ++++++++++++++++-- .../useStudyExportModalController.test.tsx | 85 +++++++-- .../src/hooks/useSingleStudyExportWorkflow.ts | 57 ++++-- 10 files changed, 419 insertions(+), 55 deletions(-) create mode 100644 packages/api/src/middleware/__tests__/omit-on-wire.test.ts diff --git a/docs/adr/0005-wire-dto-omit-on-wire.md b/docs/adr/0005-wire-dto-omit-on-wire.md index 487687fa..3d2eed2b 100644 --- a/docs/adr/0005-wire-dto-omit-on-wire.md +++ b/docs/adr/0005-wire-dto-omit-on-wire.md @@ -14,7 +14,7 @@ SampleDB API responses use a **wire DTO** layer between persistence (Drizzle/SQL | **Wire** (HTTP JSON) | Key **omitted** when unset | | **Client** (web parse / UI) | TypeScript optional / `undefined` | -Persistence-shaped rows must not be spread directly into JSON responses. Route handlers return wire-shaped payloads via `c.json(...)`; **global `omitOnWireMiddleware`** applies the null-omission policy to every successful JSON response. +Persistence-shaped rows must not be spread directly into JSON responses. Route handlers return wire-shaped payloads via `c.json(...)`; **global `omitOnWireMiddleware`** applies the null-omission policy to every successful JSON response, except a JSON Container export (see Consequences). ### Two responsibilities (do not conflate) @@ -53,6 +53,7 @@ Middleware is **permanent infrastructure** for null omission — not a substitut - Expand `@sampledb/contract/wire` schemas incrementally where discriminated shape matters; do not wire-schema every CRUD route. - Web parsers import `@sampledb/contract/wire` for container responses; optional fields use `.optional()` in Zod, not `.nullish()`. - Inbound container writes use the unified `ContainerWriteInput` shape ([ADR 0006](./0006-unified-container-inbound-write.md)) — same placement/identity split as outbound wire, not flat legacy fields. +- A JSON Container export keeps its nulls: the handler calls `keepNullsOnWire` from `packages/api/src/middleware/omit-on-wire.ts`, and every row carries every export column, null when empty. An export is a file, and a row missing a key reads as a missing column (#122). No other response opts out. - Export columns split accordingly (`barcode`, `sublabel`, `sheet_name`). Stored export configurations migrate retired keys on read (`label` → `sheet_name`; append `sublabel` when `barcode` or `sheet_name` is present). ## Related ADRs diff --git a/packages/api/src/lib/export/format.ts b/packages/api/src/lib/export/format.ts index 3536ccdc..47d83cf0 100644 --- a/packages/api/src/lib/export/format.ts +++ b/packages/api/src/lib/export/format.ts @@ -44,6 +44,17 @@ export async function formatAsCSV( return serializeCsv(columnKeys, rows, { ...options, neutralizeFormulas: true }) } +/** + * The row with every export column present: an empty value becomes null, so every row of a + * JSON export has the same keys, as every row of a CSV export has the same columns. + */ +function withEveryColumn(row: ContainerExportData, columnKeys: string[]): ContainerExportData { + const values = row as unknown as Record + const complete: Record = {} + for (const key of columnKeys) complete[key] = values[key] ?? null + return complete as unknown as ContainerExportData +} + export async function formatAsJSON( database: Database, data: ContainerExportData[], @@ -62,7 +73,9 @@ export async function formatAsJSON( containers: ContainerExportData[] }> { const columnKeys = await resolveExportColumnKeys(database, data, columns, userId) - const filteredData = filterContainerExportColumns(data, columnKeys) + const filteredData = filterContainerExportColumns(data, columnKeys).map((row) => + withEveryColumn(row, columnKeys), + ) return { export_metadata: { diff --git a/packages/api/src/middleware/__tests__/omit-on-wire.test.ts b/packages/api/src/middleware/__tests__/omit-on-wire.test.ts new file mode 100644 index 00000000..5c11dcc2 --- /dev/null +++ b/packages/api/src/middleware/__tests__/omit-on-wire.test.ts @@ -0,0 +1,26 @@ +import { describe, it, expect } from 'vitest' +import { Hono } from 'hono' +import { keepNullsOnWire, omitOnWireMiddleware } from '../omit-on-wire' + +function createApp(): Hono { + const app = new Hono() + app.use('*', omitOnWireMiddleware) + app.get('/omits', (c) => c.json({ rows: [{ id: 1, position: null }] })) + app.get('/keeps', (c) => { + keepNullsOnWire(c) + return c.json({ rows: [{ id: 1, position: null }] }) + }) + return app +} + +describe('omitOnWireMiddleware', () => { + it('omits null values from a JSON response', async () => { + const res = await createApp().request('/omits') + expect(await res.json()).toEqual({ rows: [{ id: 1 }] }) + }) + + it('sends null values when the handler calls keepNullsOnWire', async () => { + const res = await createApp().request('/keeps') + expect(await res.json()).toEqual({ rows: [{ id: 1, position: null }] }) + }) +}) diff --git a/packages/api/src/middleware/omit-on-wire.ts b/packages/api/src/middleware/omit-on-wire.ts index 1c0d8823..5c6c4ae9 100644 --- a/packages/api/src/middleware/omit-on-wire.ts +++ b/packages/api/src/middleware/omit-on-wire.ts @@ -2,12 +2,27 @@ import type { Context, Next } from 'hono' import type { ContentfulStatusCode } from 'hono/utils/http-status' import { toWireJson } from '@sampledb/contract/wire' +declare module 'hono' { + interface ContextVariableMap { + /** Set by `keepNullsOnWire`: send this response's nulls instead of omitting them. */ + keepNullsOnWire?: boolean + } +} + +/** + * Send this response's null values as they are. Only a JSON Container export calls this, + * so that every row of the file has the same keys; every other response omits nulls. + */ +export function keepNullsOnWire(c: Context): void { + c.set('keepNullsOnWire', true) +} + /** Apply omit-on-wire serialization to successful JSON response bodies. */ export async function omitOnWireMiddleware(c: Context, next: Next): Promise { await next() const res = c.res - if (res.status >= 400) { + if (res.status >= 400 || c.get('keepNullsOnWire')) { return } diff --git a/packages/api/src/routes/__tests__/export.test.ts b/packages/api/src/routes/__tests__/export.test.ts index e253e066..389ee2f3 100644 --- a/packages/api/src/routes/__tests__/export.test.ts +++ b/packages/api/src/routes/__tests__/export.test.ts @@ -6,7 +6,7 @@ import { type AuthenticatedRouteTestContext, } from '../../__tests__/helpers/authenticated-route-test' import { createExportRoutes } from '../export' -import { errorLogs, specimen as specimenTable, study as studyTable, unit as unitTable } from '../../db/schema' +import { errorLogs, micronixPlate as micronixPlateTable, specimen as specimenTable, study as studyTable, unit as unitTable } from '../../db/schema' import { SQL_IN_CHUNK_SIZE } from '../../lib/sql-helpers' import * as specimensCsv from '../../lib/export/specimens-csv' import { @@ -609,6 +609,84 @@ describe('Export API', () => { }) }) + describe('JSON export keeps every column', () => { + /** One Subject with a placed micronix tube and an unplaced Container, which has no position. */ + async function seedPlacedAndUnplaced(shortCode: string) { + const { micronixId } = await seedTypedContainers(shortCode, { cryovial: false }) + const [spec] = await ctx.db.select().from(specimenTable).limit(1) + const [unit] = await ctx.db.select().from(unitTable).limit(1) + const unplaced = await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id }) + return { micronixId, unplacedId: unplaced.id } + } + + /** Every row has the same keys, including columns that are empty on some or all rows. */ + function expectSameKeys(rows: Array>) { + expect(rows.length).toBe(2) + const keys = Object.keys(rows[0]).sort() + expect(keys).toEqual(expect.arrayContaining(['position', 'comment', 'sheet_name'])) + for (const row of rows) expect(Object.keys(row).sort()).toEqual(keys) + } + + it('gives every row of a GET JSON export the same keys, with null for an empty value', async () => { + const { unplacedId } = await seedPlacedAndUnplaced('JKEYS') + + const res = await ctx.request('/api/export/containers?study=JKEYS&format=json') + expect(res.status).toBe(200) + const body = (await res.json()) as { containers: Array> } + + expectSameKeys(body.containers) + const unplaced = body.containers.find((row) => row.container_id === unplacedId) + expect(unplaced?.position).toBeNull() + }) + + it('gives every row of a POST JSON export the same keys', async () => { + await seedPlacedAndUnplaced('PKEYS') + + const res = await ctx.request('/api/export/containers', { + method: 'POST', + json: { study: 'PKEYS', subject_names: ['Subj1'], format: 'json' }, + }) + expect(res.status).toBe(200) + const body = (await res.json()) as { data: Array> } + + expectSameKeys(body.data) + }) + + it('gives every row of a multi-study JSON export the same keys', async () => { + await seedPlacedAndUnplaced('MKEYS') + + const res = await ctx.request('/api/export/containers/multi-study', { + method: 'POST', + json: { entries: [{ study_short_code: 'MKEYS', subject_name: 'Subj1' }], format: 'json' }, + }) + expect(res.status).toBe(200) + const body = (await res.json()) as { data: Array> } + + expectSameKeys(body.data) + }) + + it('gives every row of a barcode JSON export the same keys', async () => { + await seedPlacedAndUnplaced('BKEYS') + const otherStudy = await createTestStudy(ctx.db, { title: 'Other', shortCode: 'BKEYS2' }) + const subject = await createTestStudySubject(ctx.db, { studyId: otherStudy.id, name: 'Subj2' }) + const [firstSpecimen] = await ctx.db.select().from(specimenTable).limit(1) + const spec = await createTestSpecimen(ctx.db, firstSpecimen.specimenTypeId, { studySubjectId: subject.id }) + const [unit] = await ctx.db.select().from(unitTable).limit(1) + const container = await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id }) + const [plate] = await ctx.db.select().from(micronixPlateTable).limit(1) + await createTestMicronixTube(ctx.db, { containerId: container.id, collectionId: plate.id, barcode: 'BKEYS-B' }) + + const res = await ctx.request('/api/export/containers/by-barcodes', { + method: 'POST', + json: { barcodes: ['BKEYS-MX', 'BKEYS-B'], format: 'json' }, + }) + expect(res.status).toBe(200) + const body = (await res.json()) as { data: Array> } + + expectSameKeys(body.data) + }) + }) + describe('GET /api/export/available-types', () => { it('returns 400 when study parameter is missing', async () => { const res = await ctx.request('/api/export/available-types', { method: 'GET' }) diff --git a/packages/api/src/routes/export.ts b/packages/api/src/routes/export.ts index f3c22c30..d27b4a35 100644 --- a/packages/api/src/routes/export.ts +++ b/packages/api/src/routes/export.ts @@ -27,6 +27,12 @@ import { exportSpecimensCsv } from '../lib/export/specimens-csv' import type { ExportFilters, MultiStudyExportEntry } from '../lib/export/types' import { createAuthMiddleware } from '../middleware/auth' import { idListQueryParam } from '../lib/query-params' +import { keepNullsOnWire } from '../middleware/omit-on-wire' + +/** A POST export that returns JSON rows keeps its nulls, so every row has the same keys. */ +function keepNullsInJsonRows(c: Context, result: object): void { + if ('format' in result && result.format === 'json') keepNullsOnWire(c) +} function parseTagIdsFromBody(body: unknown): number[] | undefined { if (!body || typeof body !== 'object' || !('tag_ids' in body)) return undefined @@ -195,6 +201,7 @@ export function createExportRoutes(database: Database): Hono { } if (result.kind === 'json') { + keepNullsOnWire(c) c.header('Content-Type', 'application/json') c.header('Content-Disposition', `attachment; filename="${result.filename}"`) return c.json(result.json) @@ -248,6 +255,7 @@ export function createExportRoutes(database: Database): Hono { return c.json(result) } + keepNullsInJsonRows(c, result) return c.json(result) } catch (error) { return handleExportFailure(c, 'Failed to export containers', error) @@ -326,6 +334,7 @@ export function createExportRoutes(database: Database): Hono { dateTolerance: body.date_tolerance || 0, }) + keepNullsInJsonRows(c, result) return c.json(result) } catch (error) { return handleExportFailure(c, 'Failed to export containers', error) @@ -349,6 +358,7 @@ export function createExportRoutes(database: Database): Hono { userId: user?.id, }) + keepNullsInJsonRows(c, result) return c.json(result) } catch (error) { return handleExportFailure(c, 'Failed to export containers', error) diff --git a/packages/docs/src/content/docs/guides/bulk-operations/export.md b/packages/docs/src/content/docs/guides/bulk-operations/export.md index 5a7d7882..51f8b105 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/export.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/export.md @@ -44,7 +44,7 @@ You can clear all filters without re-uploading the CSV. **XLSX download** marks identifier and code columns as text cells so leading zeros and long barcodes survive in Excel. -**JSON download** is for programmatic import. +**JSON download** is for programmatic import. Every row has the same keys; a column with no value on that row is `null`. On this page, the study export modal, and barcode export, you can override CSV delimiter, UTF-8 BOM, and line ending. Defaults (comma, BOM on, CRLF) match [CSV file guidelines](/docs/guides/troubleshooting/csv-guidelines/). diff --git a/packages/web/src/hooks/__tests__/useSingleStudyExportWorkflow.test.tsx b/packages/web/src/hooks/__tests__/useSingleStudyExportWorkflow.test.tsx index dc95fa12..7b8e01fa 100644 --- a/packages/web/src/hooks/__tests__/useSingleStudyExportWorkflow.test.tsx +++ b/packages/web/src/hooks/__tests__/useSingleStudyExportWorkflow.test.tsx @@ -1,22 +1,49 @@ import { describe, it, expect, vi, beforeEach } from 'vitest' import { renderHook, waitFor, act } from '@testing-library/react' import { exportApi } from '../../lib/api/export' -import { buildSubjectDatesFromCsv, useSingleStudyExportWorkflow } from '../useSingleStudyExportWorkflow' +import { + buildSubjectDatesFromCsv, + useSingleStudyExportWorkflow, + type SingleStudyExportOutcome, +} from '../useSingleStudyExportWorkflow' vi.mock('../../lib/export-filter-csv', () => ({ parseExportModalCsv: vi.fn(), })) -vi.mock('../../lib/api/export', () => ({ - exportApi: { - containersCount: vi.fn(), - containersCountByNames: vi.fn(), - containers: vi.fn(), - containersByNames: vi.fn(), - downloadEnvelope: vi.fn(), - downloadGetResponse: vi.fn(), - }, -})) +vi.mock('../../lib/api/export', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('export', { + exportApi: { + containersCount: vi.fn(), + containersCountByNames: vi.fn(), + containers: vi.fn(), + containersByNames: vi.fn(), + downloadEnvelope: vi.fn(), + downloadGetResponse: vi.fn(), + }, + }) +}) + +type CountResponse = Awaited> + +/** A promise the test settles by hand, to order overlapping count responses. */ +function deferred() { + let resolve!: (value: T) => void + let reject!: (reason: unknown) => void + const promise = new Promise((res, rej) => { + resolve = res + reject = rej + }) + return { promise, resolve, reject } +} + +const emptySummary = { + total_containers: 0, + subjects_with_results: [], + subjects_no_results: [], + subjects_not_found: [], +} describe('useSingleStudyExportWorkflow', () => { beforeEach(() => { @@ -103,7 +130,7 @@ describe('useSingleStudyExportWorkflow', () => { await waitFor(() => expect(result.current.count).toBe(5)) - let outcome: 'summary' | 'close' = 'summary' + let outcome: SingleStudyExportOutcome = 'summary' await act(async () => { outcome = await result.current.submitExport({ columns: ['container_id'] }) }) @@ -137,7 +164,7 @@ describe('useSingleStudyExportWorkflow', () => { await result.current.handleCSVUpload(new File(['x'], 'subjects.csv', { type: 'text/csv' })) }) - let outcome: 'summary' | 'close' = 'close' + let outcome: SingleStudyExportOutcome = 'close' await act(async () => { outcome = await result.current.submitExport({ columns: ['container_id'] }) }) @@ -149,6 +176,132 @@ describe('useSingleStudyExportWorkflow', () => { }) }) +describe('useSingleStudyExportWorkflow in CSV mode with Tags selected', () => { + beforeEach(async () => { + vi.clearAllMocks() + vi.mocked(exportApi.containersCount).mockResolvedValue({ count: 5 }) + const { parseExportModalCsv } = await import('../../lib/export-filter-csv') + vi.mocked(parseExportModalCsv).mockResolvedValue([{ subject_name: 'SUBJ-1' }]) + }) + + it('counts and exports only Containers with the selected Tags', async () => { + vi.mocked(exportApi.containersCountByNames).mockResolvedValue({ count: 1, summary: emptySummary }) + vi.mocked(exportApi.containersByNames).mockResolvedValue({ + summary: emptySummary, + data: btoa('csv-data'), + format: 'csv', + filename: 'study_ST1_export.csv', + }) + + const { result } = renderHook(() => useSingleStudyExportWorkflow({ studyCode: 'ST1', isOpen: true })) + + await act(async () => { + result.current.switchUploadMode('csv') + await result.current.handleCSVUpload(new File(['x'], 'subjects.csv', { type: 'text/csv' })) + result.current.toggleArrayFilter('tag_ids', 7) + result.current.toggleArrayFilter('tag_ids', 9) + }) + + await waitFor( + () => { + expect(exportApi.containersCountByNames).toHaveBeenLastCalledWith( + expect.objectContaining({ subject_names: ['SUBJ-1'], tag_ids: [7, 9] }), + ) + }, + { timeout: 2000 }, + ) + + await act(async () => { + await result.current.submitExport({ columns: ['container_id'] }) + }) + + expect(exportApi.containersByNames).toHaveBeenCalledWith( + expect.objectContaining({ subject_names: ['SUBJ-1'], tag_ids: [7, 9] }), + ) + }) +}) + +describe('useSingleStudyExportWorkflow count requests', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('ignores a count response that arrives after a newer one', async () => { + const older = deferred() + vi.mocked(exportApi.containersCount) + .mockReturnValueOnce(older.promise as never) + .mockResolvedValueOnce({ count: 4 }) + + const { result } = renderHook(() => useSingleStudyExportWorkflow({ studyCode: 'ST1', isOpen: true })) + await waitFor(() => expect(exportApi.containersCount).toHaveBeenCalledTimes(1)) + + await act(async () => { + await result.current.refreshCount() + }) + expect(result.current.count).toBe(4) + + await act(async () => { + older.resolve({ count: 99 }) + await older.promise + }) + + expect(result.current.count).toBe(4) + expect(result.current.loadingCount).toBe(false) + }) + + it('ignores an error from a count request a newer one replaced', async () => { + const older = deferred() + vi.mocked(exportApi.containersCount) + .mockReturnValueOnce(older.promise as never) + .mockResolvedValueOnce({ count: 4 }) + + const { result } = renderHook(() => useSingleStudyExportWorkflow({ studyCode: 'ST1', isOpen: true })) + await waitFor(() => expect(exportApi.containersCount).toHaveBeenCalledTimes(1)) + + await act(async () => { + await result.current.refreshCount() + }) + + await act(async () => { + older.reject({ response: { data: { error: 'Database busy' } } }) + await older.promise.catch(() => undefined) + }) + + expect(result.current.count).toBe(4) + expect(result.current.error).toBeNull() + }) + + it('keeps loading while the latest count request is pending', async () => { + const older = deferred() + const newer = deferred() + vi.mocked(exportApi.containersCount) + .mockReturnValueOnce(older.promise as never) + .mockReturnValueOnce(newer.promise as never) + + const { result } = renderHook(() => useSingleStudyExportWorkflow({ studyCode: 'ST1', isOpen: true })) + await waitFor(() => expect(exportApi.containersCount).toHaveBeenCalledTimes(1)) + + let latest: Promise = Promise.resolve() + act(() => { + latest = result.current.refreshCount() + }) + await act(async () => { + older.resolve({ count: 99 }) + await older.promise + }) + + expect(result.current.loadingCount).toBe(true) + expect(result.current.count).toBeNull() + + await act(async () => { + newer.resolve({ count: 4 }) + await latest + }) + expect(result.current.count).toBe(4) + expect(result.current.loadingCount).toBe(false) + }) +}) + describe('buildSubjectDatesFromCsv', () => { it('keeps every visit per subject and drops subjects that have an undated row', () => { expect( diff --git a/packages/web/src/hooks/__tests__/useStudyExportModalController.test.tsx b/packages/web/src/hooks/__tests__/useStudyExportModalController.test.tsx index cb6e9e49..3798b075 100644 --- a/packages/web/src/hooks/__tests__/useStudyExportModalController.test.tsx +++ b/packages/web/src/hooks/__tests__/useStudyExportModalController.test.tsx @@ -8,23 +8,29 @@ vi.mock('../../lib/export-filter-csv', () => ({ parseExportModalCsv: vi.fn(), })) -vi.mock('../../lib/api/export', () => ({ - exportApi: { - availableTypes: vi.fn(), - containersCount: vi.fn(), - containersCountByNames: vi.fn(), - containers: vi.fn(), - containersByNames: vi.fn(), - downloadEnvelope: vi.fn(), - downloadGetResponse: vi.fn(), - }, -})) +vi.mock('../../lib/api/export', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('export', { + exportApi: { + availableTypes: vi.fn(), + containersCount: vi.fn(), + containersCountByNames: vi.fn(), + containers: vi.fn(), + containersByNames: vi.fn(), + downloadEnvelope: vi.fn(), + downloadGetResponse: vi.fn(), + }, + }) +}) -vi.mock('../../lib/api/reference-data', () => ({ - tagsApi: { - list: vi.fn(), - }, -})) +vi.mock('../../lib/api/reference-data', async () => { + const { createMockedDomainModule } = await import('../../__tests__/helpers/mock-api') + return createMockedDomainModule('reference-data', { + tagsApi: { + list: vi.fn(), + }, + }) +}) vi.mock('../useExportConfigurations', () => ({ useExportConfigurations: () => ({ @@ -105,4 +111,51 @@ describe('useStudyExportModalController', () => { expect(exportApi.containers).toHaveBeenCalled() }) + + it('keeps the modal open and shows the error when an export fails', async () => { + vi.mocked(exportApi.containers).mockRejectedValue({ response: { data: { error: 'Failed to export containers' } } }) + const onClose = vi.fn() + + const { result } = renderHook(() => + useStudyExportModalController({ + studyCode: 'ST1', + isOpen: true, + onClose, + }), + ) + + await waitFor(() => expect(result.current.countPreview.count).toBe(3)) + + await act(async () => { + result.current.actionBar.onExport() + // Past the one-second close a successful export waits for. + await new Promise((resolve) => setTimeout(resolve, 1100)) + }) + + expect(result.current.error).toBe('Failed to export containers') + expect(onClose).not.toHaveBeenCalled() + }) + + it('closes the modal one second after a successful export', async () => { + vi.mocked(exportApi.containers).mockResolvedValue(new Blob(['csv'])) + vi.mocked(exportApi.downloadGetResponse).mockImplementation(() => undefined) + const onClose = vi.fn() + + const { result } = renderHook(() => + useStudyExportModalController({ + studyCode: 'ST1', + isOpen: true, + onClose, + }), + ) + + await waitFor(() => expect(result.current.countPreview.count).toBe(3)) + + await act(async () => { + result.current.actionBar.onExport() + }) + + await waitFor(() => expect(onClose).toHaveBeenCalledTimes(1), { timeout: 2000 }) + expect(result.current.error).toBeNull() + }) }) diff --git a/packages/web/src/hooks/useSingleStudyExportWorkflow.ts b/packages/web/src/hooks/useSingleStudyExportWorkflow.ts index 72f76296..09c7a01c 100644 --- a/packages/web/src/hooks/useSingleStudyExportWorkflow.ts +++ b/packages/web/src/hooks/useSingleStudyExportWorkflow.ts @@ -13,6 +13,12 @@ export type SingleStudyExportSummary = { errors?: string[] } +/** + * What the modal does after a submit: close once the file downloads, stay open to show the + * summary of a CSV-upload export, or stay open to show the error. + */ +export type SingleStudyExportOutcome = 'summary' | 'close' | 'error' + export type SingleStudyExportSubmitParams = { columns: string[] } @@ -55,9 +61,16 @@ export function useSingleStudyExportWorkflow(options: { const countDebounceTimerRef = useRef | null>(null) const updateCountRef = useRef<() => Promise>(() => Promise.resolve()) + // Each count request takes the next number; only the latest one may update state. + const countRequestSeqRef = useRef(0) + const updateCount = useCallback(async () => { + const requestSeq = ++countRequestSeqRef.current + const isLatest = () => requestSeq === countRequestSeqRef.current + if (uploadMode === 'csv' && csvData.length === 0) { setCount(null) + setLoadingCount(false) return } @@ -65,25 +78,26 @@ export function useSingleStudyExportWorkflow(options: { setLoadingCount(true) setError(null) - if (uploadMode === 'csv') { - const response = await exportApi.containersCountByNames({ - study: studyCode, - subject_names: csvData.map((row) => row.subject_name), - subject_dates: buildSubjectDatesFromCsv(csvData), - date_tolerance: dateTolerance, - specimen_type_ids: filters.specimen_type_ids, - container_types: filters.container_types, - date_from: filters.date_from, - date_to: filters.date_to, - created_from: filters.created_from, - created_to: filters.created_to, - }) - setCount(response.count) - } else { - const response = await exportApi.containersCount(filters) - setCount(response.count) - } + const response = + uploadMode === 'csv' + ? await exportApi.containersCountByNames({ + study: studyCode, + subject_names: csvData.map((row) => row.subject_name), + subject_dates: buildSubjectDatesFromCsv(csvData), + date_tolerance: dateTolerance, + specimen_type_ids: filters.specimen_type_ids, + container_types: filters.container_types, + tag_ids: filters.tag_ids, + date_from: filters.date_from, + date_to: filters.date_to, + created_from: filters.created_from, + created_to: filters.created_to, + }) + : await exportApi.containersCount(filters) + if (!isLatest()) return + setCount(response.count) } catch (err: unknown) { + if (!isLatest()) return console.error('Failed to get count:', err) const message = err && typeof err === 'object' && 'response' in err @@ -92,7 +106,7 @@ export function useSingleStudyExportWorkflow(options: { setError(message || 'Failed to get count') setCount(null) } finally { - setLoadingCount(false) + if (isLatest()) setLoadingCount(false) } }, [filters, uploadMode, csvData, dateTolerance, studyCode]) @@ -200,7 +214,7 @@ export function useSingleStudyExportWorkflow(options: { }, [filters]) const submitExport = useCallback( - async (params: SingleStudyExportSubmitParams): Promise<'summary' | 'close'> => { + async (params: SingleStudyExportSubmitParams): Promise => { try { setExporting(true) setError(null) @@ -216,6 +230,7 @@ export function useSingleStudyExportWorkflow(options: { columns: params.columns, specimen_type_ids: filters.specimen_type_ids, container_types: filters.container_types, + tag_ids: filters.tag_ids, date_from: filters.date_from, date_to: filters.date_to, created_from: filters.created_from, @@ -264,7 +279,7 @@ export function useSingleStudyExportWorkflow(options: { ? (err as { response?: { data?: { error?: string } } }).response?.data?.error : undefined setError(message || 'Export failed') - return 'close' + return 'error' } finally { setExporting(false) } From eff1ab96399c06a7bbe42bf3e3e4c66c85397e32 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:02:00 -0800 Subject: [PATCH 44/54] test(export): a control tube's empty study_code is null in a JSON barcode export (#122) Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/export/__tests__/container-export.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/src/lib/export/__tests__/container-export.test.ts b/packages/api/src/lib/export/__tests__/container-export.test.ts index 6185eeaf..b289b7b1 100644 --- a/packages/api/src/lib/export/__tests__/container-export.test.ts +++ b/packages/api/src/lib/export/__tests__/container-export.test.ts @@ -309,7 +309,7 @@ describe('Container export executors', () => { expect(byId.get(alphaTube.id)?.study_code).toBe('ALPHA') expect(byId.get(betaTube.id)?.study_code).toBe('BETA') expect(byId.get(controlTube.id)).toMatchObject({ control_batch_name: 'Batch 1' }) - expect(byId.get(controlTube.id)?.study_code).toBeUndefined() + expect(byId.get(controlTube.id)?.study_code).toBeNull() }) it('returns an empty export when no barcode matches', async () => { From d88b559e2f45c16d09b3ab1ca76d01a06440605f Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:07:12 -0800 Subject: [PATCH 45/54] fix(db): apply the lab's legacy data cleanup decisions in migration 013 (#126) Migration 013 carries out decisions 1, 3, 4, 5, 6 and 8 from #124: clears the four placeholder collection dates, sets the reviewed exhausted Containers to remaining 0, empties NA-style cryovial barcodes, deletes Container 34513 and empty plate 1760, and merges the two CTRL case-variant Subject pairs the way mergeSubjects does. It then adds a unique index on (study_id, name COLLATE NOCASE). Each step is guarded by a temp trigger that aborts on rows the lab did not review: counts above the snapshot's, exhausted Containers outside the reviewed id list, a placeholder micronix barcode, a changed Container 34513 or plate 1760, a renamed merge pair, or any other case-variant Subject pair. Co-Authored-By: Claude Opus 5.5 --- packages/api/initial_schema.sql | 3 +- packages/api/src/db/__tests__/client.test.ts | 11 +- .../src/db/__tests__/migration-013.test.ts | 313 ++++++++++++++++++ .../src/db/__tests__/schema-evolution.test.ts | 15 +- .../db/migrations/013_legacy_data_cleanup.sql | 169 ++++++++++ packages/api/src/db/schema-evolution.ts | 2 +- packages/api/src/db/schema.ts | 2 + 7 files changed, 505 insertions(+), 10 deletions(-) create mode 100644 packages/api/src/db/__tests__/migration-013.test.ts create mode 100644 packages/api/src/db/migrations/013_legacy_data_cleanup.sql diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index ea80d50a..94aeafb7 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -55,6 +55,7 @@ CREATE TABLE IF NOT EXISTS study_subject ( CREATE INDEX IF NOT EXISTS study_subject_study_id_idx ON study_subject(study_id);--> statement-breakpoint CREATE INDEX IF NOT EXISTS study_subject_name_idx ON study_subject(name);--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_idx ON study_subject(study_id, name);--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_nocase_idx ON study_subject(study_id, name COLLATE NOCASE);--> statement-breakpoint CREATE TABLE IF NOT EXISTS control_definition ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE, @@ -297,7 +298,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (12);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (13);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/client.test.ts b/packages/api/src/db/__tests__/client.test.ts index 3ae770fa..6b26b7c5 100644 --- a/packages/api/src/db/__tests__/client.test.ts +++ b/packages/api/src/db/__tests__/client.test.ts @@ -32,15 +32,18 @@ describe('Database Client', () => { db.exec(`CREATE TABLE micronix_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE cryovial_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE static_well (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, position TEXT)`) - db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL)`) + db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL, last_updated TEXT)`) db.exec(`CREATE TABLE sheet (id INTEGER PRIMARY KEY, name TEXT NOT NULL, box_id INTEGER, bag_id INTEGER)`) - db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, study_subject_id INTEGER, control_batch_id INTEGER, specimen_type_id INTEGER NOT NULL, collection_date DATE)`) + db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, study_subject_id INTEGER, control_batch_id INTEGER, specimen_type_id INTEGER NOT NULL, collection_date DATE, last_updated TEXT)`) // Tables migration 011 indexes - db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER)`) + db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER, comment TEXT, remaining_quantity REAL, last_updated TEXT)`) db.exec(`CREATE TABLE storage_container_tag (storage_container_id INTEGER, tag_id INTEGER)`) db.exec(`CREATE TABLE control_batch (id INTEGER PRIMARY KEY, control_definition_id INTEGER, properties TEXT)`) db.exec(`CREATE TABLE location (id INTEGER PRIMARY KEY, parent_id INTEGER, path TEXT)`) - db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER)`) + db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER, name TEXT)`) + // Tables migration 013 reads or changes + db.exec(`CREATE TABLE container_derivation (id INTEGER PRIMARY KEY, parent_container_id INTEGER, child_container_id INTEGER)`) + db.exec(`CREATE TABLE qpcr_experiment_well (id INTEGER PRIMARY KEY, storage_container_id INTEGER, specimen_id INTEGER)`) db.exec(`CREATE TABLE cryovial_box (id INTEGER PRIMARY KEY, location_id INTEGER)`) db.exec(`CREATE TABLE bag (id INTEGER PRIMARY KEY, location_id INTEGER)`) // Intentionally omit paper — migration 003 preflight creates legacy stub diff --git a/packages/api/src/db/__tests__/migration-013.test.ts b/packages/api/src/db/__tests__/migration-013.test.ts new file mode 100644 index 00000000..5b732e0d --- /dev/null +++ b/packages/api/src/db/__tests__/migration-013.test.ts @@ -0,0 +1,313 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Database } from 'bun:sqlite' +import { applyInitialSchema } from '../apply-initial-schema' +import { CURRENT_SCHEMA_VERSION, evolveOperationalSchema, getRecordedSchemaVersion } from '../schema-evolution' + +/** + * Migration 013 applies the lab's legacy data cleanup decisions (issues #124 and #126). Each test + * builds a database in production's version-12 shape: the current snapshot, without the indexes + * production lacks, seeded with the production ids each decision names. + */ + +const NOW = '2024-01-01 00:00:00' + +function databaseAtVersion12(): Database { + const sqlite = new Database(':memory:') + sqlite.exec('PRAGMA foreign_keys=ON') + applyInitialSchema(sqlite) + sqlite.exec('UPDATE schema_version SET version = 12') + // Production has neither index: no Specimen identity index (decision 7), and migration 013 + // adds the Subject name one. + sqlite.exec('DROP INDEX IF EXISTS idx_specimen_study_subject_type_date') + sqlite.exec('DROP INDEX IF EXISTS study_subject_study_name_nocase_idx') + sqlite.exec(`INSERT INTO study (id, title, short_code, is_longitudinal, lead_person, created, last_updated) VALUES + (3, 'Controls', 'CTRL', 0, 'Lab', '${NOW}', '${NOW}'), (8, 'Jiangsu', 'JIANGSU17', 0, 'Lab', '${NOW}', '${NOW}')`) + sqlite.exec(`INSERT INTO specimen_type (id, name, created, last_updated) VALUES (2, 'DBS', '${NOW}', '${NOW}'), (21, 'WB', '${NOW}', '${NOW}')`) + sqlite.exec(`INSERT INTO unit (id, symbol, name, category) VALUES (5, 'uL', 'microliter', 'volume')`) + sqlite.exec(`INSERT INTO storage_type (id, name) VALUES (1, 'Freezer')`) + sqlite.exec(`INSERT INTO location (id, name, storage_type_id) VALUES (107, 'Freezer A', 1)`) + sqlite.exec(`INSERT INTO cryovial_box (id, location_id, name, created, last_updated) VALUES (900, 107, 'Box 900', '${NOW}', '${NOW}')`) + return sqlite +} + +function addSubject(sqlite: Database, id: number, studyId: number, name: string): void { + sqlite.prepare(`INSERT INTO study_subject (id, study_id, name, created, last_updated) VALUES (?, ?, ?, ?, ?)`).run(id, studyId, name, NOW, NOW) +} + +function addSpecimen(sqlite: Database, id: number, subjectId: number, typeId: number, date: string | null): void { + sqlite + .prepare(`INSERT INTO specimen (id, study_subject_id, specimen_type_id, collection_date, created, last_updated) VALUES (?, ?, ?, ?, ?, ?)`) + .run(id, subjectId, typeId, date, NOW, NOW) +} + +function addContainer(sqlite: Database, id: number, specimenId: number, options: { comment?: string; remaining?: number } = {}): void { + sqlite + .prepare(`INSERT INTO storage_container (id, specimen_id, comment, total_quantity, remaining_quantity, unit_id, created, last_updated) VALUES (?, ?, ?, 1.0, ?, 5, ?, ?)`) + .run(id, specimenId, options.comment ?? null, options.remaining ?? 1.0, NOW, NOW) +} + +function addCryovialTube(sqlite: Database, id: number, barcode: string | null): void { + sqlite.prepare(`INSERT INTO cryovial_tube (id, collection_id, barcode) VALUES (?, 900, ?)`).run(id, barcode) +} + +function addMicronixTube(sqlite: Database, id: number, plateId: number, barcode: string): void { + sqlite.prepare(`INSERT INTO micronix_tube (id, collection_id, barcode) VALUES (?, ?, ?)`).run(id, plateId, barcode) +} + +/** Production's rows for every decision, plus neighbours each step must leave alone. */ +function seedProductionRows(sqlite: Database): void { + addSubject(sqlite, 31047, 3, '3d7_0.05') + addSubject(sqlite, 66537, 3, '3D7_0.05') + addSubject(sqlite, 31049, 3, '3d7_10k') + addSubject(sqlite, 66543, 3, '3D7_10K') + addSubject(sqlite, 25613, 8, 'JS087') + addSubject(sqlite, 500, 8, 'S500') + + // Decision 6: Specimen 43644 has no match on the target and moves to it; Specimen 43645 + // matches target Specimen 120524 (same type and date) and is merged into it. + addSpecimen(sqlite, 43644, 31047, 21, '2018-03-15') + addSpecimen(sqlite, 43645, 31047, 2, '2021-06-07') + addSpecimen(sqlite, 120523, 66537, 2, '2021-06-08') + addSpecimen(sqlite, 120524, 66537, 2, '2021-06-07') + addSpecimen(sqlite, 43646, 31049, 21, '2018-03-15') + addSpecimen(sqlite, 120529, 66543, 2, '2021-06-07') + addContainer(sqlite, 70001, 43644) + addContainer(sqlite, 70002, 43645) + addContainer(sqlite, 70003, 43645) + addContainer(sqlite, 70004, 120524) + addContainer(sqlite, 70005, 43646) + + // Decision 1: placeholder dates, and dates that stay (including a future one, decision 2). + addSpecimen(sqlite, 1, 500, 2, '1970-01-01') + addSpecimen(sqlite, 2, 500, 2, '1999-09-09') + addSpecimen(sqlite, 3, 500, 2, '1899-12-30') + addSpecimen(sqlite, 4, 500, 2, '1111-01-01') + addSpecimen(sqlite, 5, 500, 2, '1970-01-02') + addSpecimen(sqlite, 6, 500, 2, '2073-02-08') + addSpecimen(sqlite, 7, 500, 21, '1970-01-01') + + // Decision 3: exhausted Containers with remaining 1, and ones that stay. + addSpecimen(sqlite, 10, 500, 21, '2020-01-01') + addContainer(sqlite, 50465, 10, { comment: 'exhausted' }) + addContainer(sqlite, 124552, 10, { comment: 'Z32592; exhausted' }) + addContainer(sqlite, 143335, 10, { comment: 'Z20759; Zymo cleaned; exhausted' }) + addContainer(sqlite, 74034, 10, { comment: 'Reuploaded & Set to Exhausted', remaining: 0 }) + addContainer(sqlite, 74035, 10, { comment: 'half used', remaining: 0.5 }) + + // Decision 4: placeholder tube barcodes, and real barcodes that contain NA. + for (const [id, barcode] of [ + [80001, 'NA'], + [80002, 'N/A'], + [80003, 'n/a'], + [80004, ' NA '], + [80005, 'NA98'], + [80006, 'DFNA'], + [80007, null], + ] as const) { + addContainer(sqlite, id, 10) + addCryovialTube(sqlite, id, barcode) + } + + // Decision 5: Container 34513 has no tube or Paper row; its Specimen keeps its other Container. + addSpecimen(sqlite, 31863, 25613, 21, '2018-07-24') + addContainer(sqlite, 34513, 31863) + addContainer(sqlite, 34514, 31863) + addCryovialTube(sqlite, 34514, 'JS087-1') + + // Decision 8: plate 1760 is empty; plate 1761 keeps its tubes. + sqlite.exec(`INSERT INTO micronix_plate (id, location_id, name, created, last_updated) VALUES + (1760, 107, 'EXVIVO-27 ', '${NOW}', '${NOW}'), (1761, 107, 'EXVIVO-27', '${NOW}', '${NOW}')`) + addContainer(sqlite, 90001, 10) + addMicronixTube(sqlite, 90001, 1761, '1000000001') +} + +function column(sqlite: Database, sql: string): T[] { + return (sqlite.prepare(sql).all() as Array>).map((row) => Object.values(row)[0] as T) +} + +function rows(sqlite: Database, sql: string): unknown[] { + return sqlite.prepare(sql).all() +} + +describe('migration 013: legacy data cleanup', () => { + let sqlite: Database + + beforeEach(() => { + sqlite = databaseAtVersion12() + }) + + afterEach(() => { + sqlite.close() + }) + + it('applies each decision to the rows it names and leaves every other row alone', () => { + seedProductionRows(sqlite) + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + // Decision 1 + expect(rows(sqlite, 'SELECT id, collection_date FROM specimen WHERE id <= 7 ORDER BY id')).toEqual([ + { id: 1, collection_date: null }, + { id: 2, collection_date: null }, + { id: 3, collection_date: null }, + { id: 4, collection_date: null }, + { id: 5, collection_date: '1970-01-02' }, + { id: 6, collection_date: '2073-02-08' }, + { id: 7, collection_date: null }, + ]) + // Decision 3 + expect(rows(sqlite, 'SELECT id, comment, remaining_quantity FROM storage_container WHERE specimen_id = 10 AND comment IS NOT NULL ORDER BY id')).toEqual([ + { id: 50465, comment: 'exhausted', remaining_quantity: 0 }, + { id: 74034, comment: 'Reuploaded & Set to Exhausted', remaining_quantity: 0 }, + { id: 74035, comment: 'half used', remaining_quantity: 0.5 }, + { id: 124552, comment: 'Z32592; exhausted', remaining_quantity: 0 }, + { id: 143335, comment: 'Z20759; Zymo cleaned; exhausted', remaining_quantity: 0 }, + ]) + // Decision 4 + expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube WHERE id BETWEEN 80001 AND 80007 ORDER BY id')).toEqual([ + { id: 80001, barcode: null }, + { id: 80002, barcode: null }, + { id: 80003, barcode: null }, + { id: 80004, barcode: null }, + { id: 80005, barcode: 'NA98' }, + { id: 80006, barcode: 'DFNA' }, + { id: 80007, barcode: null }, + ]) + // Decision 5 + expect(column(sqlite, 'SELECT id FROM storage_container WHERE specimen_id = 31863')).toEqual([34514]) + // Decision 8 + expect(column(sqlite, `SELECT id FROM micronix_plate WHERE trim(name) = 'EXVIVO-27'`)).toEqual([1761]) + expect(column(sqlite, 'SELECT COUNT(*) FROM micronix_tube WHERE collection_id = 1761')).toEqual([1]) + // Decision 6: the app's Subject merge, run for both pairs. + expect(rows(sqlite, 'SELECT id, name FROM study_subject WHERE study_id = 3 ORDER BY id')).toEqual([ + { id: 66537, name: '3D7_0.05' }, + { id: 66543, name: '3D7_10K' }, + ]) + expect(rows(sqlite, 'SELECT id, study_subject_id FROM specimen WHERE id IN (43644, 43645, 43646, 120523, 120524, 120529) ORDER BY id')).toEqual([ + { id: 43644, study_subject_id: 66537 }, + { id: 43646, study_subject_id: 66543 }, + { id: 120523, study_subject_id: 66537 }, + { id: 120524, study_subject_id: 66537 }, + { id: 120529, study_subject_id: 66543 }, + ]) + expect(rows(sqlite, 'SELECT id, specimen_id FROM storage_container WHERE id BETWEEN 70001 AND 70005 ORDER BY id')).toEqual([ + { id: 70001, specimen_id: 43644 }, + { id: 70002, specimen_id: 120524 }, + { id: 70003, specimen_id: 120524 }, + { id: 70004, specimen_id: 120524 }, + { id: 70005, specimen_id: 43646 }, + ]) + expect(column(sqlite, 'SELECT last_updated FROM study_subject WHERE id IN (66537, 66543)').every((t) => t !== NOW)).toBe(true) + expect(column(sqlite, 'SELECT last_updated FROM specimen WHERE id = 43644')).not.toEqual([NOW]) + expect(column(sqlite, 'SELECT last_updated FROM storage_container WHERE id = 70002')).not.toEqual([NOW]) + expect(column(sqlite, `SELECT name FROM sqlite_temp_master WHERE name LIKE '__migration_013%'`)).toEqual([]) + expect(column(sqlite, 'PRAGMA foreign_key_check')).toEqual([]) + }) + + it('adds a case-insensitive unique index on Subject name within a Study', () => { + seedProductionRows(sqlite) + + evolveOperationalSchema(sqlite) + + expect(() => addSubject(sqlite, 1, 3, '3d7_10k')).toThrow(/UNIQUE constraint failed/) + expect(() => addSubject(sqlite, 2, 8, 'js087')).toThrow(/UNIQUE constraint failed/) + addSubject(sqlite, 3, 8, '3d7_10k') + expect(column(sqlite, `SELECT COUNT(*) FROM study_subject WHERE name = '3d7_10k'`)).toEqual([1]) + }) + + it('changes nothing on a database without the legacy rows', () => { + addSubject(sqlite, 1, 3, 'A') + addSpecimen(sqlite, 1, 1, 2, '2020-01-01') + addContainer(sqlite, 1, 1, { comment: 'exhausted', remaining: 0 }) + addCryovialTube(sqlite, 1, 'CV1') + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(rows(sqlite, 'SELECT id, study_subject_id, collection_date FROM specimen')).toEqual([{ id: 1, study_subject_id: 1, collection_date: '2020-01-01' }]) + expect(rows(sqlite, 'SELECT id, remaining_quantity FROM storage_container')).toEqual([{ id: 1, remaining_quantity: 0 }]) + expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube')).toEqual([{ id: 1, barcode: 'CV1' }]) + expect(column(sqlite, 'SELECT COUNT(*) FROM study_subject')).toEqual([1]) + }) + + const aborts: Array<[string, (sqlite: Database) => void, RegExp]> = [ + [ + 'an exhausted Container the lab did not review still has quantity', + (db) => addContainer(db, 99999, 10, { comment: 'exhausted' }), + /exhausted/, + ], + [ + 'a reviewed exhausted Container has a remaining quantity other than 1', + (db) => db.exec('UPDATE storage_container SET remaining_quantity = 0.5 WHERE id = 50465'), + /exhausted/, + ], + [ + 'there are more Specimens on a placeholder date than the lab reviewed', + (db) => addSpecimen(db, 99999, 500, 21, '1111-01-01'), + /placeholder collection date/, + ], + [ + 'a micronix tube has a placeholder barcode, which cannot be emptied', + (db) => { + addContainer(db, 99999, 10) + addMicronixTube(db, 99999, 1761, 'N/A') + }, + /placeholder barcode/, + ], + [ + 'Container 34513 has a tube row', + (db) => addCryovialTube(db, 34513, 'X34513'), + /Container 34513/, + ], + [ + 'Container 34513 is the only Container of its Specimen', + (db) => db.exec('DELETE FROM cryovial_tube WHERE id = 34514; DELETE FROM storage_container WHERE id = 34514'), + /Container 34513/, + ], + [ + 'plate 1760 holds a tube', + (db) => { + addContainer(db, 99999, 10) + addMicronixTube(db, 99999, 1760, '1000000002') + }, + /plate 1760/, + ], + [ + 'plate 1760 has been renamed', + (db) => db.exec(`UPDATE micronix_plate SET name = 'EXVIVO-28' WHERE id = 1760`), + /plate 1760/, + ], + [ + 'a Subject to merge has been renamed', + (db) => db.exec(`UPDATE study_subject SET name = '3d7_10k_b' WHERE id = 31049`), + /merge/, + ], + [ + 'a Subject to merge has lost its target', + (db) => db.exec(`UPDATE specimen SET study_subject_id = 31049 WHERE study_subject_id = 66543; DELETE FROM study_subject WHERE id = 66543`), + /merge/, + ], + [ + 'another Study has Subjects whose names differ only by case', + (db) => { + addSubject(db, 600, 8, 'abc') + addSubject(db, 601, 8, 'ABC') + }, + /differ only by case/, + ], + ] + + for (const [situation, change, message] of aborts) { + it(`aborts, changing nothing, when ${situation}`, () => { + seedProductionRows(sqlite) + change(sqlite) + const before = sqlite.serialize() + + expect(() => evolveOperationalSchema(sqlite)).toThrow(message) + + expect(getRecordedSchemaVersion(sqlite)).toBe(12) + expect(Buffer.from(sqlite.serialize()).equals(Buffer.from(before))).toBe(true) + }) + } +}) diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index f3d3da92..c7cc7796 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -65,6 +65,13 @@ function createTablesForLaterMigrations(sqlite: Database): void { ensureColumns(sqlite, 'micronix_plate', ['location_id INTEGER']) ensureColumns(sqlite, 'cryovial_box', ['location_id INTEGER']) ensureColumns(sqlite, 'bag', ['location_id INTEGER']) + // Migration 013 reads or changes these columns. + ensureColumns(sqlite, 'study_subject', ['last_updated TEXT']) + ensureColumns(sqlite, 'specimen', ['last_updated TEXT']) + ensureColumns(sqlite, 'storage_container', ['comment TEXT', 'remaining_quantity REAL', 'last_updated TEXT']) + ensureColumns(sqlite, 'micronix_plate', ['name TEXT']) + ensureColumns(sqlite, 'container_derivation', ['parent_container_id INTEGER', 'child_container_id INTEGER']) + ensureColumns(sqlite, 'qpcr_experiment_well', ['storage_container_id INTEGER', 'specimen_id INTEGER']) // Before version 3, migration 003 renames barcode to sublabel (and its preflight creates the // table if missing), so only a database already past 003 gets paper in its later shape. const version = hasSchemaVersionTable(sqlite) ? getRecordedSchemaVersion(sqlite) : null @@ -282,13 +289,13 @@ describe('schema evolution', () => { sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') sqlite.exec('INSERT INTO schema_version (version) VALUES (4)') createTablesForLaterMigrations(sqlite) - sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (1, 1, ' S1 '), (2, 1, 's1'), (3, 2, 'S1')`) + sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (1, 1, ' S1 '), (2, 1, 'S2'), (3, 2, 'S1')`) evolveOperationalSchema(sqlite) expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) const names = sqlite.prepare('SELECT name FROM study_subject ORDER BY id').all() as Array<{ name: string }> - expect(names.map((r) => r.name)).toEqual(['S1', 's1', 'S1']) + expect(names.map((r) => r.name)).toEqual(['S1', 'S2', 'S1']) expect(() => sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (4, 1, 'S1')`)).toThrow(/UNIQUE/) sqlite.close() }) @@ -348,7 +355,7 @@ describe('schema evolution', () => { [6, null], [7, ' '], [8, ' 3/17/22 '], - [9, '1970-01-01'], + [9, '1970-01-02'], ] const insert = sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (?, 1, ?)') for (const [id, date] of values) insert.run(id, date) @@ -366,7 +373,7 @@ describe('schema evolution', () => { null, null, '2022-03-17', - '1970-01-01', + '1970-01-02', ]) sqlite.close() }) diff --git a/packages/api/src/db/migrations/013_legacy_data_cleanup.sql b/packages/api/src/db/migrations/013_legacy_data_cleanup.sql new file mode 100644 index 00000000..9113b8d0 --- /dev/null +++ b/packages/api/src/db/migrations/013_legacy_data_cleanup.sql @@ -0,0 +1,169 @@ +-- Migration 013: apply the lab's legacy data cleanup decisions (issue #124, "Decisions (2026-11-01)"; +-- issue #126). Each step changes only the rows its decision names, found by the decision's own rule +-- and checked against what the production snapshot of 2026-10-31 held. If a database holds rows the +-- lab did not review, the migration aborts and changes nothing, so they can be reviewed first. +-- 1. Placeholder collection dates (1970-01-01, 1999-09-09, 1899-12-30, 1111-01-01) become unknown. +-- 3. Containers commented "exhausted" with remaining quantity 1 get remaining quantity 0; the +-- comment stays. +-- 4. NA / N/A-style cryovial tube barcodes become empty (NULL, as for any tube without a barcode). +-- 5. Container 34513 (no tube or Paper row; Specimen 31863 keeps its other Container) is deleted. +-- 6. CTRL Subjects 3d7_0.05 (31047) and 3d7_10k (31049) are merged into 3D7_0.05 (66537) and +-- 3D7_10K (66543) the way the app's Subject merge does it, and Subject names become unique +-- within a Study ignoring case. +-- 8. Empty plate 1760 `EXVIVO-27 ` (trailing space) is deleted; plate 1761 `EXVIVO-27` stays. +-- Decisions 2 (future dates) and 7 (duplicate Specimen identity) change no data; the integrity +-- report lists them. +CREATE TEMP TABLE __migration_013_exhausted (id INTEGER PRIMARY KEY);--> statement-breakpoint +INSERT INTO __migration_013_exhausted (id) VALUES + (50465), (87730), (87732), (87735), (87736), (87737), (87738), (87739), (87741), (87742), + (87743), (87744), (87747), (87748), (87749), (87750), (87751), (87752), (87753), (87754), + (87756), (87757), (87759), (87762), (87763), (87765), (87766), (87768), (87769), (124552), + (124564), (124565), (124568), (124570), (124598), (124645), (124646), (136248), (140979), (143331), + (143335), (143339), (143343), (143347), (144277), (144318);--> statement-breakpoint +CREATE TEMP TABLE __migration_013_placeholder_dates (collection_date TEXT PRIMARY KEY, reviewed INTEGER NOT NULL);--> statement-breakpoint +INSERT INTO __migration_013_placeholder_dates (collection_date, reviewed) VALUES + ('1970-01-01', 17043), ('1999-09-09', 245), ('1899-12-30', 17), ('1111-01-01', 1);--> statement-breakpoint +CREATE TEMP TABLE __migration_013_subject_merge ( + source_id INTEGER PRIMARY KEY, + source_name TEXT NOT NULL, + target_id INTEGER NOT NULL, + target_name TEXT NOT NULL +);--> statement-breakpoint +INSERT INTO __migration_013_subject_merge (source_id, source_name, target_id, target_name) VALUES + (31047, '3d7_0.05', 66537, '3D7_0.05'), (31049, '3d7_10k', 66543, '3D7_10K');--> statement-breakpoint +-- A pair is merged only while its source Subject still exists. +DELETE FROM __migration_013_subject_merge WHERE source_id NOT IN (SELECT id FROM study_subject);--> statement-breakpoint +CREATE TEMP TABLE IF NOT EXISTS __migration_013_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint +CREATE TRIGGER __migration_013_dates_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot clear placeholder collection dates: more Specimens are dated 1970-01-01, 1999-09-09, 1899-12-30 or 1111-01-01 than the lab reviewed (17043, 245, 17 and 1). Review the newer ones first: SELECT id, collection_date, created FROM specimen WHERE collection_date IN (''1970-01-01'', ''1999-09-09'', ''1899-12-30'', ''1111-01-01'') ORDER BY created DESC.') + WHERE EXISTS ( + SELECT 1 FROM specimen s JOIN __migration_013_placeholder_dates p ON p.collection_date = s.collection_date + GROUP BY p.collection_date HAVING COUNT(*) > MAX(p.reviewed) + ); +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_exhausted_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot set exhausted Containers to remaining quantity 0: a Container commented exhausted has remaining quantity above 0 and is not one of the 46 the lab reviewed, or its remaining quantity is not 1. Review it first: SELECT id, comment, remaining_quantity FROM storage_container WHERE comment LIKE ''%exhausted%'' AND remaining_quantity > 0.') + WHERE EXISTS ( + SELECT 1 FROM storage_container + WHERE comment LIKE '%exhausted%' AND remaining_quantity > 0 + AND (id NOT IN (SELECT id FROM __migration_013_exhausted) OR remaining_quantity <> 1) + ); +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_barcodes_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot empty placeholder barcodes: a micronix tube has an NA-style placeholder barcode (micronix tubes must have one), or more cryovial tubes have one than the 285 the lab reviewed. Review them first.') + WHERE EXISTS ( + SELECT 1 FROM micronix_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' + ) OR ( + SELECT COUNT(*) FROM cryovial_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' + ) > 285; +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_container_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot delete Container 34513: it no longer matches what the lab reviewed (Specimen 31863, no tube, Paper or well row, no derivation, Tag or qPCR well, and its Specimen has another Container). Review it first.') + WHERE EXISTS (SELECT 1 FROM storage_container WHERE id = 34513) + AND NOT ( + (SELECT specimen_id FROM storage_container WHERE id = 34513) = 31863 + AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM cryovial_tube WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM paper WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM static_well WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM container_derivation WHERE parent_container_id = 34513 OR child_container_id = 34513) + AND NOT EXISTS (SELECT 1 FROM storage_container_tag WHERE storage_container_id = 34513) + AND NOT EXISTS (SELECT 1 FROM qpcr_experiment_well WHERE storage_container_id = 34513) + AND EXISTS (SELECT 1 FROM storage_container WHERE specimen_id = 31863 AND id <> 34513) + ); +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_plate_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot delete plate 1760: it no longer matches what the lab reviewed (named "EXVIVO-27 " with a trailing space, holding no tubes or wells, beside plate 1761 "EXVIVO-27"). Review it first.') + WHERE EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1760) + AND NOT ( + (SELECT name FROM micronix_plate WHERE id = 1760) = 'EXVIVO-27 ' + AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE collection_id = 1760) + AND NOT EXISTS (SELECT 1 FROM static_well WHERE collection_id = 1760) + AND EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1761 AND name = 'EXVIVO-27') + ); +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_merge_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot merge the CTRL Subjects 3d7_0.05 into 3D7_0.05 and 3d7_10k into 3D7_10K: one of them no longer has the name or Study the lab reviewed, or its target Subject is missing. Review them first.') + WHERE EXISTS ( + SELECT 1 FROM __migration_013_subject_merge m + JOIN study_subject src ON src.id = m.source_id + LEFT JOIN study_subject tgt ON tgt.id = m.target_id + WHERE src.name IS NOT m.source_name + OR tgt.name IS NOT m.target_name + OR tgt.study_id IS NOT src.study_id + OR src.study_id IS NOT (SELECT id FROM study WHERE short_code = 'CTRL') + ); +END;--> statement-breakpoint +CREATE TRIGGER __migration_013_subject_case_tr BEFORE INSERT ON __migration_013_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot make Subject names unique ignoring case: a Study has Subjects whose names differ only by case, beyond the two CTRL pairs the lab chose to merge. Merge or rename them first. SELECT study_id, group_concat(id || '':'' || name, '', '') FROM study_subject GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 lists them.') + WHERE EXISTS ( + SELECT 1 FROM study_subject + WHERE id NOT IN (SELECT source_id FROM __migration_013_subject_merge) + GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 + ); +END;--> statement-breakpoint +INSERT INTO __migration_013_guard DEFAULT VALUES;--> statement-breakpoint +DROP TRIGGER __migration_013_dates_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_exhausted_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_barcodes_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_container_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_plate_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_merge_tr;--> statement-breakpoint +DROP TRIGGER __migration_013_subject_case_tr;--> statement-breakpoint +DROP TABLE __migration_013_guard;--> statement-breakpoint +-- Decision 1 +UPDATE specimen SET collection_date = NULL +WHERE collection_date IN (SELECT collection_date FROM __migration_013_placeholder_dates);--> statement-breakpoint +-- Decision 3 +UPDATE storage_container SET remaining_quantity = 0 +WHERE id IN (SELECT id FROM __migration_013_exhausted) AND comment LIKE '%exhausted%' AND remaining_quantity = 1;--> statement-breakpoint +-- Decision 4 +UPDATE cryovial_tube SET barcode = NULL +WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA';--> statement-breakpoint +-- Decision 5 +DELETE FROM storage_container WHERE id = 34513;--> statement-breakpoint +-- Decision 8 +DELETE FROM micronix_plate WHERE id = 1760;--> statement-breakpoint +-- Decision 6, as mergeSubjects in src/lib/subjects/merge.ts does it. Each source Specimen in turn +-- (lowest id first) is merged into the first target Specimen with the same Specimen type and +-- collection date, moving its Containers there; a source Specimen with no such match moves to +-- the target Subject, where a later source Specimen can match it. Runs after decision 1, so it +-- compares the cleaned dates. +CREATE TEMP TABLE __migration_013_specimen_merge AS +SELECT s.id AS specimen_id, m.target_id, + COALESCE( + (SELECT MIN(t.id) FROM specimen t + WHERE t.study_subject_id = m.target_id AND t.specimen_type_id = s.specimen_type_id AND t.collection_date IS s.collection_date), + (SELECT MIN(e.id) FROM specimen e + WHERE e.study_subject_id = m.source_id AND e.id < s.id AND e.specimen_type_id = s.specimen_type_id AND e.collection_date IS s.collection_date) + ) AS match_id +FROM specimen s +JOIN __migration_013_subject_merge m ON m.source_id = s.study_subject_id;--> statement-breakpoint +UPDATE storage_container +SET specimen_id = (SELECT match_id FROM __migration_013_specimen_merge WHERE specimen_id = storage_container.specimen_id), + last_updated = datetime('now') +WHERE specimen_id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +UPDATE qpcr_experiment_well +SET specimen_id = (SELECT match_id FROM __migration_013_specimen_merge WHERE specimen_id = qpcr_experiment_well.specimen_id) +WHERE specimen_id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +DELETE FROM specimen +WHERE id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +UPDATE specimen +SET study_subject_id = (SELECT target_id FROM __migration_013_specimen_merge WHERE specimen_id = specimen.id), + last_updated = datetime('now') +WHERE id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NULL);--> statement-breakpoint +UPDATE study_subject SET last_updated = datetime('now') +WHERE id IN (SELECT target_id FROM __migration_013_subject_merge);--> statement-breakpoint +DELETE FROM study_subject WHERE id IN (SELECT source_id FROM __migration_013_subject_merge);--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_nocase_idx ON study_subject(study_id, name COLLATE NOCASE);--> statement-breakpoint +DROP TABLE __migration_013_specimen_merge;--> statement-breakpoint +DROP TABLE __migration_013_subject_merge;--> statement-breakpoint +DROP TABLE __migration_013_placeholder_dates;--> statement-breakpoint +DROP TABLE __migration_013_exhausted; diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index 400fe525..476c0eea 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -8,7 +8,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 12 +export const CURRENT_SCHEMA_VERSION = 13 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/db/schema.ts b/packages/api/src/db/schema.ts index 0a4c2300..506d7689 100644 --- a/packages/api/src/db/schema.ts +++ b/packages/api/src/db/schema.ts @@ -110,6 +110,8 @@ export const studySubject = sqliteTable('study_subject', { studyIdIdx: index('study_subject_study_id_idx').on(table.studyId), nameIdx: index('study_subject_name_idx').on(table.name), studyNameUniq: uniqueIndex('study_subject_study_name_idx').on(table.studyId, table.name), + // Subject names are unique within a Study ignoring case (CONTEXT.md, Subject; migration 013). + studyNameNocaseUniq: uniqueIndex('study_subject_study_name_nocase_idx').on(table.studyId, sql`${table.name} COLLATE NOCASE`), })) // Control Production Hierarchy From 30ea366afc3eb99499ede4fd673edc8a471e33ed Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:09:15 -0800 Subject: [PATCH 46/54] fix(subjects): Subject names compare ignoring case within a Study (#126) Create, bulk create and validate, rename, and Combined import refuse a Subject name that differs from one the study already has only by case, with "Subject name 'abc-1' differs only by case from Subject 'ABC-1' in this study". validateSubjectName compares with COLLATE NOCASE, matching migration 013's unique index, and batch duplicate checks fold ASCII case the same way. Rename now goes through validateSubjectName, so a Subject can still change the case of its own name. CONTEXT.md records the Subject name rule (#124 decision 6) and Specimen identity (decision 7), including that fresh installs have idx_specimen_study_subject_type_date and production does not. The user guide says names ignore case. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 6 +- packages/api/src/lib/validation.ts | 34 ++++-- .../api/src/routes/__tests__/imports.test.ts | 47 ++++++++ .../api/src/routes/__tests__/subjects.test.ts | 100 ++++++++++++++++++ packages/api/src/routes/subjects.ts | 32 ++---- .../docs/guides/bulk-operations/import.md | 3 +- .../guides/troubleshooting/best-practices.md | 2 +- .../guides/workflows/subjects-specimens.md | 2 +- 8 files changed, 190 insertions(+), 36 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 3ec656f3..26cfb14b 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -57,11 +57,15 @@ The top-level organizational unit for a research project, clinical trial, or oth _Avoid_: Project (unless referring to non-study work), trial (as a synonym for study) **Subject**: -A participant or sample source within a study, identified by a unique name within that study. +A participant or sample source within a study, identified by a unique name within that study. Names compare ignoring case (ASCII letters, as SQLite's `NOCASE` does): a study can't have both `ABC-1` and `abc-1`, and creating, renaming or importing a Subject whose name differs from an existing one only by case is refused. The unique index `study_subject_study_name_nocase_idx` enforces it. Decided in issue #124 (decision 6), where the two legacy CTRL pairs were merged by migration 013. _Avoid_: Patient (unless the study is clinical and that is your org's term), donor (unless that is your org's term) **Specimen**: The collected unit — a registered biological material from a specific collection event. One collection event, one specimen record. + +**Specimen identity**: +A Subject's Specimen is identified by its Subject, Specimen type and collection date: **Specimen registration** adds Containers to the Specimen with that combination instead of creating a second one. Existing duplicates stay as they are (issue #124, decision 7): the lab chose no merges and no new unique index on production. On the 2026-10-31 production snapshot that leaves 15 dated pairs, which migration 008 created by converting legacy `M/D/YY` dates, and undated groups: 98 before migration 013, and 502 after it cleared the placeholder collection dates (decision 1), because a Specimen dated `1970-01-01` now shares an empty date with its undated twin. The integrity report lists them all. Known difference: fresh installs have the unique index `idx_specimen_study_subject_type_date` on (`study_subject_id`, `specimen_type_id`, `collection_date`) and production does not. That index treats an empty collection date as distinct, so fresh installs also allow undated duplicates. +_Avoid_: Assuming the database refuses a duplicate Specimen; on production only registration's lookup prevents one. _Avoid_: Sample (as the domain term; see **Sample** below) **Sample**: diff --git a/packages/api/src/lib/validation.ts b/packages/api/src/lib/validation.ts index 5f345207..f8322948 100644 --- a/packages/api/src/lib/validation.ts +++ b/packages/api/src/lib/validation.ts @@ -4,12 +4,16 @@ import { eq, and, like, sql } from 'drizzle-orm' import { resolveStudyByShortCode, resolveSubjectByNameAndStudy, resolveSpecimenTypeByName } from './identifier-resolution' /** - * Validate that a subject name is unique within a study + * Validate that a subject name is unique within a study. Names compare ignoring case + * (CONTEXT.md, Subject), as the `study_subject_study_name_nocase_idx` index does, so a name that + * differs from an existing one only by case is refused. Pass `excludeSubjectId` when renaming, + * so a Subject may change the case of its own name. */ export async function validateSubjectName( database: Database, studyId: number, - name: string + name: string, + options: { excludeSubjectId?: number } = {} ): Promise<{ valid: boolean; error?: string }> { const trimmedName = name.trim() @@ -21,23 +25,39 @@ export async function validateSubjectName( return { valid: false, error: 'Subject name cannot exceed 255 characters' } } - // Check for existing subject with same name in same study + // COLLATE NOCASE matches the unique index, so this refuses exactly what the index would. const existing = await database - .select({ id: studySubject.id }) + .select({ id: studySubject.id, name: studySubject.name }) .from(studySubject) .where(and( eq(studySubject.studyId, studyId), - eq(studySubject.name, trimmedName) - ) as any) + sql`${studySubject.name} = ${trimmedName} COLLATE NOCASE`, + options.excludeSubjectId === undefined ? undefined : sql`${studySubject.id} != ${options.excludeSubjectId}` + )) .get() if (existing) { - return { valid: false, error: `Subject name '${trimmedName}' already exists in this study` } + return { valid: false, error: subjectNameTakenMessage(trimmedName, existing.name) } } return { valid: true } } +/** The error for a Subject name the study already has, exactly or differing only by case. */ +export function subjectNameTakenMessage(name: string, existingName: string): string { + return name === existingName + ? `Subject name '${name}' already exists in this study` + : `Subject name '${name}' differs only by case from Subject '${existingName}' in this study. Subject names in a study ignore case.` +} + +/** + * The key under which two Subject names in one study count as the same name. Folds ASCII letters + * only, as SQLite's NOCASE collation does. + */ +export function subjectNameKey(studyId: number, name: string): string { + return `${studyId}:${name.trim().replace(/[A-Z]/g, (letter) => letter.toLowerCase())}` +} + function isoDate(year: number, month: number, day: number): string | null { const d = new Date(Date.UTC(year, month - 1, day)) if (d.getUTCFullYear() !== year || d.getUTCMonth() !== month - 1 || d.getUTCDate() !== day) return null diff --git a/packages/api/src/routes/__tests__/imports.test.ts b/packages/api/src/routes/__tests__/imports.test.ts index c7d64927..a44926de 100644 --- a/packages/api/src/routes/__tests__/imports.test.ts +++ b/packages/api/src/routes/__tests__/imports.test.ts @@ -7,6 +7,7 @@ import { import { createImportsRoutes } from '../imports' import { createTestStudy, + createTestStudySubject, createTestSpecimenType, createTestLocation, createTestStorageType, @@ -156,6 +157,52 @@ describe('Imports API', () => { expect(afterSpecimens.length).toBe(beforeSpecimens.length) }) + it('refuses a Subject name that differs from one the study has only by case, and imports nothing', async () => { + const studyRecord = await createTestStudy(ctx.db, { title: 'Case Import', shortCode: 'CASEIMP' }) + await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'ABC-1' }) + await createTestSpecimenType(ctx.db, { name: 'Whole Blood' }) + + const res = await ctx.request('/api/imports/bulk-combined', { + method: 'POST', + json: { + studyShortCode: 'CASEIMP', + atomicMode: 'full_file', + subjects: [ + { subjectName: 'NEW-1', specimens: [{ specimenTypeName: 'Whole Blood', collectionDate: '2025-01-01' }] }, + { subjectName: 'abc-1', specimens: [{ specimenTypeName: 'Whole Blood', collectionDate: '2025-01-02' }] }, + ], + }, + }) + + expect(res.status).toBe(400) + expect(JSON.stringify(await res.json())).toContain( + "Subject name 'abc-1' differs only by case from Subject 'ABC-1' in this study. Subject names in a study ignore case.", + ) + expect((await ctx.db.select().from(studySubject)).map((s) => s.name)).toEqual(['ABC-1']) + expect(await ctx.db.select().from(specimen)).toHaveLength(0) + }) + + it('refuses two Subjects in one file whose names differ only by case', async () => { + await createTestStudy(ctx.db, { title: 'Case Import', shortCode: 'CASEIMP' }) + await createTestSpecimenType(ctx.db, { name: 'Whole Blood' }) + + const res = await ctx.request('/api/imports/bulk-combined', { + method: 'POST', + json: { + studyShortCode: 'CASEIMP', + atomicMode: 'full_file', + subjects: [ + { subjectName: 'Pair-1', specimens: [{ specimenTypeName: 'Whole Blood', collectionDate: '2025-01-01' }] }, + { subjectName: 'PAIR-1', specimens: [{ specimenTypeName: 'Whole Blood', collectionDate: '2025-01-02' }] }, + ], + }, + }) + + expect(res.status).toBe(400) + expect(JSON.stringify(await res.json())).toContain("Subject name 'PAIR-1' differs only by case from Subject 'Pair-1'") + expect(await ctx.db.select().from(studySubject)).toHaveLength(0) + }) + it('per_subject mode allows partial success and returns indexed errors', async () => { await createTestStudy(ctx.db, { title: 'Import Study2', shortCode: 'IMPBULK2' }) await createTestSpecimenType(ctx.db, { name: 'Plasma' }) diff --git a/packages/api/src/routes/__tests__/subjects.test.ts b/packages/api/src/routes/__tests__/subjects.test.ts index d8655534..fe952ddc 100644 --- a/packages/api/src/routes/__tests__/subjects.test.ts +++ b/packages/api/src/routes/__tests__/subjects.test.ts @@ -112,6 +112,106 @@ describe('Subjects API', () => { }) }) + describe('Subject names ignore case within a Study', () => { + const caseError = "Subject name 'abc-1' differs only by case from Subject 'ABC-1' in this study. Subject names in a study ignore case." + + async function studyWithSubject() { + const studyRecord = await createTestStudy(ctx.db, { title: 'Case Study', shortCode: 'CASE' }) + const subject = await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'ABC-1' }) + return { studyRecord, subject } + } + + it('POST /api/subjects refuses a name that differs from an existing one only by case', async () => { + const { studyRecord } = await studyWithSubject() + + const res = await ctx.request('/api/subjects', { method: 'POST', json: { studyId: studyRecord.id, name: ' abc-1 ' } }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe(caseError) + expect((await ctx.db.select().from(studySubject)).map((s) => s.name)).toEqual(['ABC-1']) + }) + + it('POST /api/subjects accepts the same name in a different study', async () => { + await studyWithSubject() + const other = await createTestStudy(ctx.db, { title: 'Other Study', shortCode: 'OTHER' }) + + const res = await ctx.request('/api/subjects', { method: 'POST', json: { studyId: other.id, name: 'abc-1' } }) + + expect(res.status).toBe(201) + }) + + it('POST /api/subjects/bulk refuses a case variant of an existing name and creates nothing', async () => { + await studyWithSubject() + + const res = await ctx.request('/api/subjects/bulk', { + method: 'POST', + json: { subjects: [{ studyShortCode: 'CASE', name: 'NEW-1' }, { studyShortCode: 'CASE', name: 'abc-1' }] }, + }) + + expect(res.status).toBe(400) + const data = (await res.json()) as { errors: Array<{ index: number; error: string }> } + expect(data.errors).toEqual([{ index: 1, error: caseError }]) + expect((await ctx.db.select().from(studySubject)).map((s) => s.name)).toEqual(['ABC-1']) + }) + + it('POST /api/subjects/bulk refuses two names in one batch that differ only by case', async () => { + await createTestStudy(ctx.db, { title: 'Case Study', shortCode: 'CASE' }) + + const res = await ctx.request('/api/subjects/bulk', { + method: 'POST', + json: { subjects: [{ studyShortCode: 'CASE', name: 'Dup' }, { studyShortCode: 'CASE', name: 'DUP' }] }, + }) + + expect(res.status).toBe(400) + const data = (await res.json()) as { errors: Array<{ index: number; error: string }> } + expect(data.errors).toEqual([{ index: 1, error: "Duplicate subject name 'DUP' in study" }]) + expect(await ctx.db.select().from(studySubject)).toHaveLength(0) + }) + + it('POST /api/subjects/bulk/validate reports case variants, in the batch and against the study', async () => { + await studyWithSubject() + + const res = await ctx.request('/api/subjects/bulk/validate', { + method: 'POST', + json: { + subjects: [ + { studyShortCode: 'CASE', name: 'abc-1' }, + { studyShortCode: 'CASE', name: 'Same' }, + { studyShortCode: 'CASE', name: 'SAME' }, + ], + }, + }) + + expect(res.status).toBe(200) + const data = (await res.json()) as { valid: boolean; errors: Array<{ index: number; message: string }> } + expect(data.valid).toBe(false) + expect(data.errors).toEqual([ + { index: 0, message: caseError }, + { index: 1, message: "Duplicate subject name 'SAME' in study" }, + ]) + }) + + it('PUT /api/subjects/:id refuses a rename to a case variant of another Subject', async () => { + const { studyRecord } = await studyWithSubject() + const other = await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'XYZ-2' }) + + const res = await ctx.request(`/api/subjects/${other.id}`, { method: 'PUT', json: { name: 'abc-1' } }) + + expect(res.status).toBe(400) + expect(((await res.json()) as { error: string }).error).toBe(caseError) + expect((await ctx.db.select().from(studySubject)).map((s) => s.name).sort()).toEqual(['ABC-1', 'XYZ-2']) + }) + + it('PUT /api/subjects/:id lets a Subject change the case of its own name', async () => { + const { subject } = await studyWithSubject() + + const res = await ctx.request(`/api/subjects/${subject.id}`, { method: 'PUT', json: { name: 'abc-1' } }) + + expect(res.status).toBe(200) + expect(((await res.json()) as { subject: { name: string } }).subject.name).toBe('abc-1') + }) + }) + describe('POST /api/subjects/bulk', () => { it('creates all subjects in one transaction and returns 201', async () => { await createTestStudy(ctx.db, { title: 'Bulk Study', shortCode: 'BULK' }) diff --git a/packages/api/src/routes/subjects.ts b/packages/api/src/routes/subjects.ts index 6fb6d62a..29e66bb4 100644 --- a/packages/api/src/routes/subjects.ts +++ b/packages/api/src/routes/subjects.ts @@ -15,7 +15,7 @@ import { dateRangeConditions } from '../lib/sql-helpers' import { timestampFilter } from '../lib/date-filters' import { z } from 'zod' import { resolveStudyByShortCode, resolveSubjectByNameAndStudy, resolveSpecimenTypeByName } from '../lib/identifier-resolution' -import { validateSubjectName, validateStudyShortCode, validateSpecimenData, validateCollectionDate, validateContainerTypeForSpecimenType, validateUnitForContainerType } from '../lib/validation' +import { subjectNameKey, validateSubjectName, validateStudyShortCode, validateSpecimenData, validateCollectionDate, validateContainerTypeForSpecimenType, validateUnitForContainerType } from '../lib/validation' import { getDefaultUnit, getDefaultTotalQuantity, getDefaultRemainingQuantity } from '../lib/defaults' import { resolveCollection, resolveCollectionByName } from '../lib/collections/collection-resolve' import { findExistingStudySpecimen } from '../lib/specimen-helpers' @@ -241,7 +241,7 @@ subjects.post('/bulk', memberMiddleware, async (c) => { const duplicateErrors: Array<{ index: number; error: string }> = [] for (let i = 0; i < validSubjects.length; i++) { - const key = `${validSubjects[i].studyId}:${validSubjects[i].name}` + const key = subjectNameKey(validSubjects[i].studyId, validSubjects[i].name) if (seen.has(key)) { duplicateErrors.push({ index: i, @@ -325,7 +325,7 @@ subjects.post('/bulk/validate', memberMiddleware, async (c) => { } const seen = new Set() for (let i = 0; i < validSubjects.length; i++) { - const key = `${validSubjects[i].studyId}:${validSubjects[i].name}` + const key = subjectNameKey(validSubjects[i].studyId, validSubjects[i].name) if (seen.has(key)) { errors.push({ index: i, message: `Duplicate subject name '${validSubjects[i].name}' in study` }) } @@ -417,28 +417,10 @@ subjects.put('/:id', memberMiddleware, async (c) => { const data = schema.parse(body) const trimmedName = data.name.trim() - // Validate name length - if (trimmedName.length === 0) { - return c.json({ error: 'Subject name cannot be empty' }, 400) - } - - if (trimmedName.length > 255) { - return c.json({ error: 'Subject name cannot exceed 255 characters' }, 400) - } - - // Check for duplicate name within the same study (excluding current subject) - if (trimmedName !== existingSubject.name) { - const duplicate = await dbInstance - .select({ id: studySubject.id }) - .from(studySubject) - .where(and( - eq(studySubject.studyId, existingSubject.studyId), - eq(studySubject.name, trimmedName) - ) as any) - .get() - - if (duplicate) { - return c.json({ error: `Subject name '${trimmedName}' already exists in this study` }, 400) + if (trimmedName !== existingSubject.name) { + const nameValidation = await validateSubjectName(dbInstance, existingSubject.studyId, trimmedName, { excludeSubjectId: id }) + if (!nameValidation.valid) { + throw new ValidationError(nameValidation.error ?? 'Invalid subject name') } } diff --git a/packages/docs/src/content/docs/guides/bulk-operations/import.md b/packages/docs/src/content/docs/guides/bulk-operations/import.md index cec25ae6..aa14a6e5 100644 --- a/packages/docs/src/content/docs/guides/bulk-operations/import.md +++ b/packages/docs/src/content/docs/guides/bulk-operations/import.md @@ -9,7 +9,7 @@ You can also open **Bulk import** from **More actions** on a study detail page. ## Import types -**Subjects Only.** Creates subjects in studies that already exist. Required columns: `study_short_code` and `subject_name`. Subject names must be unique within each study. +**Subjects Only.** Creates subjects in studies that already exist. Required columns: `study_short_code` and `subject_name`. Subject names must be unique within each study, ignoring case: `ABC-1` and `abc-1` are the same name. **Specimens Only.** Adds specimens to subjects that already exist. You must select a **Container Type**. Choose **No Containers** to register specimens without creating containers. If a subject name is missing, that row fails so you do not create a near-duplicate subject. @@ -98,6 +98,7 @@ In a longitudinal study, use one CSV row per collection. Each row creates a sepa - **Missing required columns.** Add the columns named in the error. For the collection identifier, use `plate_name` for micronix or static wells, `box_name` for cryovial tubes, or exactly one of `box_name` or `bag_name` for paper. - **Invalid study codes / Study not found.** Create the study first, or correct the code. - **Subject name already exists / Duplicate subject.** For Combined, existing subjects get new specimens. For Subjects Only, each name must be new in the study and unique in the batch. +- **Subject name differs only by case.** Names compare ignoring case, so `abc-1` is refused when the study has `ABC-1`. Spell the name as the study already does, or choose a different name. - **Missing collections.** Assign locations on **Create Missing Collections**, or fix names and barcodes in the CSV. - **Location cannot contain collections.** Pick a location that is allowed to hold collections, such as a freezer or shelf configured for that. - **Position already used / duplicate position.** That well is taken in the plate or box, or the same position appears twice in the file for the same collection. diff --git a/packages/docs/src/content/docs/guides/troubleshooting/best-practices.md b/packages/docs/src/content/docs/guides/troubleshooting/best-practices.md index 5c01f4f6..1e406c30 100644 --- a/packages/docs/src/content/docs/guides/troubleshooting/best-practices.md +++ b/packages/docs/src/content/docs/guides/troubleshooting/best-practices.md @@ -13,7 +13,7 @@ These are constraints in the product, not optional style. ## Subjects and specimens -- Subject names are unique **within a study**. +- Subject names are unique **within a study**, ignoring case: `ABC-1` and `abc-1` can't both exist in one study. - Specimen type names in import and forms must match **Reference Data** exactly, including capitalization. - Collection dates use `YYYY-MM-DD`. The registration form does not default to today if you leave the date blank. - Specimens cannot be edited in the app after create. diff --git a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md index f24a5fa7..da572654 100644 --- a/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md +++ b/packages/docs/src/content/docs/guides/workflows/subjects-specimens.md @@ -3,7 +3,7 @@ title: Subjects and specimens description: Create subjects and register specimens one at a time --- -A **subject** is a participant or sample source in a study. The name must be unique within that study. A **specimen** is the registered biological sample from one collection event. One subject can have many specimens. Each specimen belongs to one source, usually a subject. +A **subject** is a participant or sample source in a study. The name must be unique within that study, ignoring case, so a study can't have both `ABC-1` and `abc-1`. A **specimen** is the registered biological sample from one collection event. One subject can have many specimens. Each specimen belongs to one source, usually a subject. This page covers one-at-a-time entry. For many rows, see [Bulk import](/docs/guides/bulk-operations/import/). From e95bd6fc64ba6143fece1299a0b6737db37b7a9b Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:14:14 -0800 Subject: [PATCH 47/54] feat(data-audit): report legacy data problems in the integrity report (#126) The integrity report adds seven checks, each with a section on the report page and a count on the overview: - duplicate Specimen identity (Subject, Specimen type and collection date, dated or undated) - Subject names in one Study that differ only by case - Collection names that differ only by case or surrounding spaces (sheets within their box or bag) - Containers in a Container type their Specimen type does not allow - Containers commented "exhausted" with remaining quantity above 0 - the four known placeholder collection dates - collection dates after today (#124 decision 2) The duplicate-barcode check already covers micronix and cryovial tubes (#105); the overview card no longer calls it micronix-only. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/data-audit.ts | 260 +++++++++++++++++- packages/api/src/lib/validation.ts | 14 +- .../src/routes/__tests__/data-audit.test.ts | 196 +++++++++++++ .../docs/guides/advanced/data-integrity.md | 7 + packages/web/src/lib/api/admin.ts | 52 ++++ .../src/pages/AdminDataIntegrityOverview.tsx | 11 +- .../src/pages/AdminDataIntegrityReport.tsx | 237 +++++++++++++++- ...dminDataIntegrityEmptyCollections.test.tsx | 7 + .../AdminDataIntegrityOverview.test.tsx | 29 ++ .../AdminDataIntegrityReport.test.tsx | 87 ++++++ 10 files changed, 889 insertions(+), 11 deletions(-) diff --git a/packages/api/src/lib/data-audit.ts b/packages/api/src/lib/data-audit.ts index 5ae7c5c5..e4628abc 100644 --- a/packages/api/src/lib/data-audit.ts +++ b/packages/api/src/lib/data-audit.ts @@ -18,9 +18,12 @@ import { storageContainerTag, paper, tag, + specimenType, + specimenTypeContainerType, } from '../db/schema' -import { eq, inArray, isNotNull, sql } from 'drizzle-orm' +import { and, asc, eq, gt, inArray, isNotNull, sql } from 'drizzle-orm' import { isIsoCalendarDate } from './date-filters' +import { foldNameCase, localToday, subjectNameKey } from './validation' import { isStoredGridPosition, type GridCollectionType } from '@sampledb/contract' export type CollectionType = 'micronix_plate' | 'cryovial_box' | 'box' | 'bag' @@ -360,6 +363,63 @@ export interface ContainerWithInvalidGridPositionItem { position: string } +/** + * Subject Specimens that share a Subject, Specimen type and collection date (CONTEXT.md, + * Specimen identity). The lab chose to keep the legacy groups (issue #124, decision 7). + */ +export interface DuplicateSpecimenIdentityItem { + studyId: number + studyShortCode: string + subjectId: number + subjectName: string + specimenTypeId: number + specimenTypeName: string + /** Absent for undated Specimens. */ + collectionDate?: string + specimenIds: number[] +} + +/** Subjects in one Study whose names differ only by case (CONTEXT.md, Subject). */ +export interface CaseVariantSubjectNameItem { + studyId: number + studyShortCode: string + subjects: Array<{ id: number; name: string }> +} + +/** + * Collections of one type whose names differ only by case or surrounding spaces. Sheets are + * compared within their box or bag. + */ +export interface CaseVariantCollectionNameItem { + type: CollectionType | 'sheet' + /** The box or bag holding the sheets; absent for other Collection types. */ + parent?: { type: 'box' | 'bag'; id: number } + collections: Array<{ id: number; name: string }> +} + +export type ContainerType = 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' + +/** Containers whose Container type is not one their Specimen type allows. */ +export interface ContainerInDisallowedContainerTypeItem { + id: number + containerType: ContainerType + specimenId: number + specimenTypeId: number + specimenTypeName: string +} + +/** Containers commented "exhausted" whose remaining quantity is above 0, so they show as In Use. */ +export interface ExhaustedContainerWithRemainingQuantityItem { + id: number + comment: string + remainingQuantity: number +} + +export interface SpecimenCollectionDateItem { + id: number + collectionDate: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -375,6 +435,13 @@ export interface IntegrityReport { containersWithNoGridPosition: ContainerWithNoGridPositionItem[] specimensWithNonIsoCollectionDate: SpecimenWithNonIsoCollectionDateItem[] containersWithInvalidGridPosition: ContainerWithInvalidGridPositionItem[] + duplicateSpecimenIdentities: DuplicateSpecimenIdentityItem[] + caseVariantSubjectNames: CaseVariantSubjectNameItem[] + caseVariantCollectionNames: CaseVariantCollectionNameItem[] + containersInDisallowedContainerType: ContainerInDisallowedContainerTypeItem[] + exhaustedContainersWithRemainingQuantity: ExhaustedContainerWithRemainingQuantityItem[] + specimensWithPlaceholderCollectionDate: SpecimenCollectionDateItem[] + specimensWithFutureCollectionDate: SpecimenCollectionDateItem[] } /** Collections (plates, boxes, bags) whose locationId does not exist in location. */ @@ -611,6 +678,176 @@ export async function listContainersWithInvalidGridPosition(db: Database): Promi return results } +/** Group `items` by `key`, keeping only groups with more than one item, in first-seen order. */ +function groupsOfMany(items: T[], key: (item: T) => string): T[][] { + const groups = new Map() + for (const item of items) { + const k = key(item) + const group = groups.get(k) + if (group) group.push(item) + else groups.set(k, [item]) + } + return [...groups.values()].filter((group) => group.length > 1) +} + +/** + * Subject Specimens that share a Subject, Specimen type and collection date, including undated + * ones (CONTEXT.md, Specimen identity). Production has no unique index on this combination. + */ +export async function listDuplicateSpecimenIdentities(db: Database): Promise { + const groups = await db + .select({ + subjectId: specimen.studySubjectId, + specimenTypeId: specimen.specimenTypeId, + collectionDate: specimen.collectionDate, + ids: sql`group_concat(${specimen.id})`, + }) + .from(specimen) + .where(isNotNull(specimen.studySubjectId)) + .groupBy(specimen.studySubjectId, specimen.specimenTypeId, specimen.collectionDate) + .having(sql`COUNT(*) > 1`) + .orderBy(asc(specimen.studySubjectId), asc(specimen.specimenTypeId), asc(specimen.collectionDate)) + if (groups.length === 0) return [] + const subjectIds = [...new Set(groups.map((g) => g.subjectId!))] + const subjects = new Map( + ( + await db + .select({ id: studySubject.id, name: studySubject.name, studyId: studySubject.studyId, shortCode: study.shortCode }) + .from(studySubject) + .leftJoin(study, eq(study.id, studySubject.studyId)) + .where(inArray(studySubject.id, subjectIds)) + ).map((row) => [row.id, row]), + ) + const typeNames = new Map((await db.select({ id: specimenType.id, name: specimenType.name }).from(specimenType)).map((row) => [row.id, row.name])) + return groups.map((group) => { + const subject = subjects.get(group.subjectId!) + return { + studyId: subject?.studyId ?? 0, + studyShortCode: subject?.shortCode ?? '', + subjectId: group.subjectId!, + subjectName: subject?.name ?? '', + specimenTypeId: group.specimenTypeId, + specimenTypeName: typeNames.get(group.specimenTypeId) ?? '', + ...(group.collectionDate == null ? {} : { collectionDate: String(group.collectionDate) }), + specimenIds: group.ids.split(',').map(Number).sort((a, b) => a - b), + } + }) +} + +/** Subjects in one Study whose names differ only by case, which the Subject name rule forbids. */ +export async function listCaseVariantSubjectNames(db: Database): Promise { + const subjects = await db + .select({ id: studySubject.id, name: studySubject.name, studyId: studySubject.studyId, shortCode: study.shortCode }) + .from(studySubject) + .leftJoin(study, eq(study.id, studySubject.studyId)) + .orderBy(asc(studySubject.studyId), asc(studySubject.id)) + return groupsOfMany(subjects, (s) => subjectNameKey(s.studyId, s.name)).map((group) => ({ + studyId: group[0]!.studyId, + studyShortCode: group[0]!.shortCode ?? '', + subjects: group.map((s) => ({ id: s.id, name: s.name })), + })) +} + +/** + * Collections of one type (sheets: within one box or bag) whose names differ only by case or + * surrounding spaces. Names spelled exactly alike are not listed: plate, box and bag names are + * unique, and a bag may hold two sheets of the same name. + */ +export async function listCaseVariantCollectionNames(db: Database): Promise { + const topLevel: Array<{ type: CollectionType; rows: Array<{ id: number; name: string }> }> = [ + { type: 'micronix_plate', rows: await db.select({ id: micronixPlate.id, name: micronixPlate.name }).from(micronixPlate).orderBy(micronixPlate.id) }, + { type: 'cryovial_box', rows: await db.select({ id: cryovialBox.id, name: cryovialBox.name }).from(cryovialBox).orderBy(cryovialBox.id) }, + { type: 'box', rows: await db.select({ id: box.id, name: box.name }).from(box).orderBy(box.id) }, + { type: 'bag', rows: await db.select({ id: bag.id, name: bag.name }).from(bag).orderBy(bag.id) }, + ] + const results: CaseVariantCollectionNameItem[] = [] + const hasVariants = (group: Array<{ name: string }>) => new Set(group.map((c) => c.name)).size > 1 + for (const { type, rows } of topLevel) { + for (const group of groupsOfMany(rows, (c) => foldNameCase(c.name)).filter(hasVariants)) { + results.push({ type, collections: group.map((c) => ({ id: c.id, name: c.name })) }) + } + } + const sheets = await db.select({ id: sheet.id, name: sheet.name, boxId: sheet.boxId, bagId: sheet.bagId }).from(sheet).orderBy(sheet.id) + for (const group of groupsOfMany(sheets, (s) => `${s.boxId ?? ''}:${s.bagId ?? ''}:${foldNameCase(s.name)}`).filter(hasVariants)) { + const first = group[0]! + const parent = first.boxId != null ? { type: 'box' as const, id: first.boxId } : first.bagId != null ? { type: 'bag' as const, id: first.bagId } : undefined + results.push({ + type: 'sheet', + ...(parent ? { parent } : {}), + collections: group.map((s) => ({ id: s.id, name: s.name })), + }) + } + return results +} + +/** Containers whose Container type their Specimen type does not allow (specimen_type_container_type). */ +export async function listContainersInDisallowedContainerType(db: Database): Promise { + const sources = [ + { containerType: 'micronix_tube', table: micronixTube }, + { containerType: 'cryovial_tube', table: cryovialTube }, + { containerType: 'paper', table: paper }, + { containerType: 'static_well', table: staticWell }, + ] as const + const results: ContainerInDisallowedContainerTypeItem[] = [] + for (const { containerType, table } of sources) { + const rows = await db + .select({ + id: storageContainer.id, + specimenId: specimen.id, + specimenTypeId: specimenType.id, + specimenTypeName: specimenType.name, + }) + .from(table) + .innerJoin(storageContainer, eq(storageContainer.id, table.id)) + .innerJoin(specimen, eq(specimen.id, storageContainer.specimenId)) + .innerJoin(specimenType, eq(specimenType.id, specimen.specimenTypeId)) + .where( + sql`NOT EXISTS (SELECT 1 FROM ${specimenTypeContainerType} WHERE ${specimenTypeContainerType.specimenTypeId} = ${specimen.specimenTypeId} AND ${specimenTypeContainerType.containerType} = ${containerType})`, + ) + for (const row of rows) results.push({ ...row, containerType }) + } + return results.sort((a, b) => a.id - b.id) +} + +/** Containers commented "exhausted" (any case) whose remaining quantity is above 0. */ +export async function listExhaustedContainersWithRemainingQuantity(db: Database): Promise { + const rows = await db + .select({ id: storageContainer.id, comment: storageContainer.comment, remainingQuantity: storageContainer.remainingQuantity }) + .from(storageContainer) + .where(and(sql`${storageContainer.comment} LIKE '%exhausted%'`, gt(storageContainer.remainingQuantity, 0))) + .orderBy(storageContainer.id) + return rows.map((row) => ({ id: row.id, comment: row.comment ?? '', remainingQuantity: row.remainingQuantity ?? 0 })) +} + +/** + * Collection dates the legacy application stored for "unknown". Migration 013 cleared them + * (issue #124, decision 1); the report flags any that come back. + */ +export const PLACEHOLDER_COLLECTION_DATES = ['1970-01-01', '1999-09-09', '1899-12-30', '1111-01-01'] as const + +/** Specimens dated with a known placeholder collection date. */ +export async function listSpecimensWithPlaceholderCollectionDate(db: Database): Promise { + const rows = await db + .select({ id: specimen.id, collectionDate: specimen.collectionDate }) + .from(specimen) + .where(inArray(specimen.collectionDate, [...PLACEHOLDER_COLLECTION_DATES])) + .orderBy(specimen.id) + return rows.map((row) => ({ id: row.id, collectionDate: String(row.collectionDate) })) +} + +/** + * Specimens with a YYYY-MM-DD collection date after today on the server. New dates in the future + * are refused on write; the lab corrects the legacy ones by hand (issue #124, decision 2). + */ +export async function listSpecimensWithFutureCollectionDate(db: Database): Promise { + const rows = await db + .select({ id: specimen.id, collectionDate: specimen.collectionDate }) + .from(specimen) + .where(and(sql`date(${specimen.collectionDate}) IS ${specimen.collectionDate}`, gt(specimen.collectionDate, localToday()))) + .orderBy(specimen.id) + return rows.map((row) => ({ id: row.id, collectionDate: String(row.collectionDate) })) +} + /** Build full integrity report (all checks). */ export async function getIntegrityReport(db: Database): Promise { const [ @@ -628,6 +865,13 @@ export async function getIntegrityReport(db: Database): Promise containersWithNoGridPosition, specimensWithNonIsoCollectionDate, containersWithInvalidGridPosition, + duplicateSpecimenIdentities, + caseVariantSubjectNames, + caseVariantCollectionNames, + containersInDisallowedContainerType, + exhaustedContainersWithRemainingQuantity, + specimensWithPlaceholderCollectionDate, + specimensWithFutureCollectionDate, ] = await Promise.all([ listEmptyCollections(db), listCollectionsWithMissingLocation(db), @@ -643,6 +887,13 @@ export async function getIntegrityReport(db: Database): Promise listContainersWithNoGridPosition(db), listSpecimensWithNonIsoCollectionDate(db), listContainersWithInvalidGridPosition(db), + listDuplicateSpecimenIdentities(db), + listCaseVariantSubjectNames(db), + listCaseVariantCollectionNames(db), + listContainersInDisallowedContainerType(db), + listExhaustedContainersWithRemainingQuantity(db), + listSpecimensWithPlaceholderCollectionDate(db), + listSpecimensWithFutureCollectionDate(db), ]) return { emptyCollections, @@ -659,5 +910,12 @@ export async function getIntegrityReport(db: Database): Promise containersWithNoGridPosition, specimensWithNonIsoCollectionDate, containersWithInvalidGridPosition, + duplicateSpecimenIdentities, + caseVariantSubjectNames, + caseVariantCollectionNames, + containersInDisallowedContainerType, + exhaustedContainersWithRemainingQuantity, + specimensWithPlaceholderCollectionDate, + specimensWithFutureCollectionDate, } } diff --git a/packages/api/src/lib/validation.ts b/packages/api/src/lib/validation.ts index f8322948..8f49be9a 100644 --- a/packages/api/src/lib/validation.ts +++ b/packages/api/src/lib/validation.ts @@ -50,12 +50,14 @@ export function subjectNameTakenMessage(name: string, existingName: string): str : `Subject name '${name}' differs only by case from Subject '${existingName}' in this study. Subject names in a study ignore case.` } -/** - * The key under which two Subject names in one study count as the same name. Folds ASCII letters - * only, as SQLite's NOCASE collation does. - */ +/** `name` trimmed and with ASCII letters lower-cased, as SQLite's NOCASE collation compares. */ +export function foldNameCase(name: string): string { + return name.trim().replace(/[A-Z]/g, (letter) => letter.toLowerCase()) +} + +/** The key under which two Subject names in one study count as the same name. */ export function subjectNameKey(studyId: number, name: string): string { - return `${studyId}:${name.trim().replace(/[A-Z]/g, (letter) => letter.toLowerCase())}` + return `${studyId}:${foldNameCase(name)}` } function isoDate(year: number, month: number, day: number): string | null { @@ -65,7 +67,7 @@ function isoDate(year: number, month: number, day: number): string | null { } /** Today's calendar date on the server, as YYYY-MM-DD. */ -function localToday(): string { +export function localToday(): string { const now = new Date() return isoDate(now.getFullYear(), now.getMonth() + 1, now.getDate())! } diff --git a/packages/api/src/routes/__tests__/data-audit.test.ts b/packages/api/src/routes/__tests__/data-audit.test.ts index 0f069d2f..029b0d33 100644 --- a/packages/api/src/routes/__tests__/data-audit.test.ts +++ b/packages/api/src/routes/__tests__/data-audit.test.ts @@ -21,6 +21,11 @@ import { createTestCryovialBox, createTestCryovialTube, createTestMicronixTube, + createTestBox, + createTestBag, + createTestSheet, + createTestPaper, + createTestSpecimenTypeContainerType, } from '../../__tests__/helpers/factories' import { micronixTube, @@ -250,6 +255,13 @@ describe('Data audit API', () => { containersWithNoGridPosition: unknown[] specimensWithNonIsoCollectionDate: unknown[] containersWithInvalidGridPosition: unknown[] + duplicateSpecimenIdentities: unknown[] + caseVariantSubjectNames: unknown[] + caseVariantCollectionNames: unknown[] + containersInDisallowedContainerType: unknown[] + exhaustedContainersWithRemainingQuantity: unknown[] + specimensWithPlaceholderCollectionDate: unknown[] + specimensWithFutureCollectionDate: unknown[] } expect(Array.isArray(data.emptyCollections)).toBe(true) expect(Array.isArray(data.collectionsWithMissingLocation)).toBe(true) @@ -265,6 +277,13 @@ describe('Data audit API', () => { expect(Array.isArray(data.containersWithNoGridPosition)).toBe(true) expect(Array.isArray(data.specimensWithNonIsoCollectionDate)).toBe(true) expect(data.containersWithInvalidGridPosition).toEqual([]) + expect(data.duplicateSpecimenIdentities).toEqual([]) + expect(data.caseVariantSubjectNames).toEqual([]) + expect(data.caseVariantCollectionNames).toEqual([]) + expect(data.containersInDisallowedContainerType).toEqual([]) + expect(data.exhaustedContainersWithRemainingQuantity).toEqual([]) + expect(data.specimensWithPlaceholderCollectionDate).toEqual([]) + expect(data.specimensWithFutureCollectionDate).toEqual([]) }) it('flags specimens whose collection date is not YYYY-MM-DD', async () => { @@ -520,6 +539,183 @@ describe('Data audit API', () => { expect(match!.collectionId).toBe(plate.id) }) + async function report(key: K): Promise { + const res = await ctx.request('/api/admin/data-audit/integrity-report', { method: 'GET' }) + expect(res.status).toBe(200) + return ((await res.json()) as Record)[key] + } + + it('lists Specimens that share a Subject, Specimen type and collection date, dated or not', async () => { + // Production has no Specimen identity index (CONTEXT.md, Specimen identity); fresh installs do. + ctx.sqlite.exec('DROP INDEX idx_specimen_study_subject_type_date') + const studyRecord = await createTestStudy(ctx.db, { title: 'Identity Study', shortCode: 'IDS' }) + const subject = await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'ID-1' }) + const blood = await createTestSpecimenType(ctx.db, { name: 'STIdBlood' }) + const plasma = await createTestSpecimenType(ctx.db, { name: 'STIdPlasma' }) + const dated = [ + await createTestSpecimen(ctx.db, blood.id, { studySubjectId: subject.id, collectionDate: '2022-03-17' }), + await createTestSpecimen(ctx.db, blood.id, { studySubjectId: subject.id, collectionDate: '2022-03-17' }), + ] + const undated = [ + await createTestSpecimen(ctx.db, plasma.id, { studySubjectId: subject.id }), + await createTestSpecimen(ctx.db, plasma.id, { studySubjectId: subject.id }), + await createTestSpecimen(ctx.db, plasma.id, { studySubjectId: subject.id }), + ] + await createTestSpecimen(ctx.db, blood.id, { studySubjectId: subject.id, collectionDate: '2022-03-18' }) + await createTestSpecimen(ctx.db, blood.id, { studySubjectId: subject.id }) + + expect(await report('duplicateSpecimenIdentities')).toEqual([ + { + studyId: studyRecord.id, + studyShortCode: 'IDS', + subjectId: subject.id, + subjectName: 'ID-1', + specimenTypeId: blood.id, + specimenTypeName: 'STIdBlood', + collectionDate: '2022-03-17', + specimenIds: dated.map((s) => s.id), + }, + { + studyId: studyRecord.id, + studyShortCode: 'IDS', + subjectId: subject.id, + subjectName: 'ID-1', + specimenTypeId: plasma.id, + specimenTypeName: 'STIdPlasma', + specimenIds: undated.map((s) => s.id), + }, + ]) + }) + + it('lists Subjects in one Study whose names differ only by case', async () => { + ctx.sqlite.exec('DROP INDEX study_subject_study_name_nocase_idx') + const ctrl = await createTestStudy(ctx.db, { title: 'Controls', shortCode: 'CTRL' }) + const other = await createTestStudy(ctx.db, { title: 'Other', shortCode: 'OTH' }) + const upper = await createTestStudySubject(ctx.db, { studyId: ctrl.id, name: '3D7_10K' }) + const lower = await createTestStudySubject(ctx.db, { studyId: ctrl.id, name: '3d7_10k' }) + await createTestStudySubject(ctx.db, { studyId: other.id, name: '3d7_10K' }) + await createTestStudySubject(ctx.db, { studyId: ctrl.id, name: '3D7_0.05' }) + + expect(await report('caseVariantSubjectNames')).toEqual([ + { + studyId: ctrl.id, + studyShortCode: 'CTRL', + subjects: [ + { id: upper.id, name: '3D7_10K' }, + { id: lower.id, name: '3d7_10k' }, + ], + }, + ]) + }) + + it('lists Collections of one type whose names differ only by case or surrounding spaces', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'ShelfCase', description: '' }) + const loc = await createTestLocation(ctx.db, { name: 'LocCase', storageTypeId: String(storageType.id), canContainCollections: true }) + const plateA = await createTestMicronixPlate(ctx.db, { name: 'EXVIVO-27', locationId: loc.id }) + const plateB = await createTestMicronixPlate(ctx.db, { name: 'EXVIVO-27 ', locationId: loc.id }) + await createTestCryovialBox(ctx.db, { name: 'exvivo-27', locationId: loc.id }) + const boxA = await createTestBox(ctx.db, { name: 'Box-1', locationId: loc.id }) + const boxB = await createTestBox(ctx.db, { name: 'BOX-1', locationId: loc.id }) + const bag = await createTestBag(ctx.db, { name: 'Bag-1', locationId: loc.id }) + // Two sheets with the same spelling in one bag are allowed (migration 006), so they are not listed. + await createTestSheet(ctx.db, { name: 'S1', bagId: bag.id }) + await createTestSheet(ctx.db, { name: 'S1', bagId: bag.id }) + const sheetA = await createTestSheet(ctx.db, { name: 'Sheet-2', boxId: boxA.id }) + const sheetB = await createTestSheet(ctx.db, { name: 'sheet-2', boxId: boxA.id }) + await createTestSheet(ctx.db, { name: 'SHEET-2', boxId: boxB.id }) + + expect(await report('caseVariantCollectionNames')).toEqual([ + { type: 'micronix_plate', collections: [{ id: plateA.id, name: 'EXVIVO-27' }, { id: plateB.id, name: 'EXVIVO-27 ' }] }, + { type: 'box', collections: [{ id: boxA.id, name: 'Box-1' }, { id: boxB.id, name: 'BOX-1' }] }, + { type: 'sheet', parent: { type: 'box', id: boxA.id }, collections: [{ id: sheetA.id, name: 'Sheet-2' }, { id: sheetB.id, name: 'sheet-2' }] }, + ]) + }) + + it('lists Containers whose Container type their Specimen type does not allow', async () => { + const storageType = await createTestStorageType(ctx.db, { name: 'ShelfAllow', description: '' }) + const loc = await createTestLocation(ctx.db, { name: 'LocAllow', storageTypeId: String(storageType.id), canContainCollections: true }) + const plate = await createTestMicronixPlate(ctx.db, { name: 'PlateAllow', locationId: loc.id }) + const cvBox = await createTestCryovialBox(ctx.db, { name: 'BoxAllow', locationId: loc.id }) + const box1 = await createTestBox(ctx.db, { name: 'PaperBoxAllow', locationId: loc.id }) + const sheet1 = await createTestSheet(ctx.db, { name: 'SheetAllow', boxId: box1.id }) + const antibody = await createTestSpecimenType(ctx.db, { name: 'Polyclonal Ab' }) + const dbs = await createTestSpecimenType(ctx.db, { name: 'DBS Allow' }) + await createTestSpecimenTypeContainerType(ctx.db, { specimenTypeId: dbs.id, containerType: 'paper' }) + await createTestSpecimenTypeContainerType(ctx.db, { specimenTypeId: dbs.id, containerType: 'cryovial_tube' }) + const unit = await createTestUnit(ctx.db, { symbol: 'uL-allow', name: 'microliter', category: 'volume' }) + const abSpecimen = await createTestSpecimen(ctx.db, antibody.id) + const dbsSpecimen = await createTestSpecimen(ctx.db, dbs.id) + const abTube = await createTestStorageContainer(ctx.db, { specimenId: abSpecimen.id, unitId: unit.id }) + await createTestMicronixTube(ctx.db, { containerId: abTube.id, collectionId: plate.id, barcode: 'AB-1', position: null }) + const dbsTube = await createTestStorageContainer(ctx.db, { specimenId: dbsSpecimen.id, unitId: unit.id }) + await createTestMicronixTube(ctx.db, { containerId: dbsTube.id, collectionId: plate.id, barcode: 'DBS-1', position: null }) + const dbsCryo = await createTestStorageContainer(ctx.db, { specimenId: dbsSpecimen.id, unitId: unit.id }) + await createTestCryovialTube(ctx.db, { containerId: dbsCryo.id, collectionId: cvBox.id, barcode: 'DBS-2', position: null }) + const dbsPaper = await createTestStorageContainer(ctx.db, { specimenId: dbsSpecimen.id, unitId: unit.id }) + await createTestPaper(ctx.db, { containerId: dbsPaper.id, sheetId: sheet1.id }) + + expect(await report('containersInDisallowedContainerType')).toEqual([ + { id: abTube.id, containerType: 'micronix_tube', specimenId: abSpecimen.id, specimenTypeId: antibody.id, specimenTypeName: 'Polyclonal Ab' }, + { id: dbsTube.id, containerType: 'micronix_tube', specimenId: dbsSpecimen.id, specimenTypeId: dbs.id, specimenTypeName: 'DBS Allow' }, + ]) + }) + + it('lists Containers commented exhausted that still have remaining quantity', async () => { + const specType = await createTestSpecimenType(ctx.db, { name: 'STExhausted' }) + const spec = await createTestSpecimen(ctx.db, specType.id) + const unit = await createTestUnit(ctx.db, { symbol: 'uL-ex', name: 'microliter', category: 'volume' }) + const container = async (comment: string | null, remaining: number) => { + const row = await createTestStorageContainer(ctx.db, { specimenId: spec.id, unitId: unit.id }) + await ctx.db.update(storageContainer).set({ comment, remainingQuantity: remaining }).where(eq(storageContainer.id, row.id)) + return row.id + } + const plain = await container('exhausted', 1) + const tagged = await container('Z20329; Exhausted', 0.25) + await container('exhausted', 0) + await container('in use', 1) + await container(null, 1) + + expect(await report('exhaustedContainersWithRemainingQuantity')).toEqual([ + { id: plain, comment: 'exhausted', remainingQuantity: 1 }, + { id: tagged, comment: 'Z20329; Exhausted', remainingQuantity: 0.25 }, + ]) + }) + + it('lists Specimens on a known placeholder collection date', async () => { + const specType = await createTestSpecimenType(ctx.db, { name: 'STPlaceholder' }) + const ids: number[] = [] + for (const date of ['1970-01-01', '1999-09-09', '1899-12-30', '1111-01-01']) { + ids.push((await createTestSpecimen(ctx.db, specType.id, { collectionDate: date })).id) + } + await createTestSpecimen(ctx.db, specType.id, { collectionDate: '1970-01-02' }) + await createTestSpecimen(ctx.db, specType.id) + + expect(await report('specimensWithPlaceholderCollectionDate')).toEqual([ + { id: ids[0], collectionDate: '1970-01-01' }, + { id: ids[1], collectionDate: '1999-09-09' }, + { id: ids[2], collectionDate: '1899-12-30' }, + { id: ids[3], collectionDate: '1111-01-01' }, + ]) + }) + + it('lists Specimens whose collection date is after today', async () => { + const specType = await createTestSpecimenType(ctx.db, { name: 'STFuture' }) + const future = await createTestSpecimen(ctx.db, specType.id, { collectionDate: '2073-02-08' }) + const tomorrow = new Date() + tomorrow.setDate(tomorrow.getDate() + 1) + const tomorrowIso = `${tomorrow.getFullYear()}-${String(tomorrow.getMonth() + 1).padStart(2, '0')}-${String(tomorrow.getDate()).padStart(2, '0')}` + const next = await createTestSpecimen(ctx.db, specType.id, { collectionDate: tomorrowIso }) + const today = new Date() + const todayIso = `${today.getFullYear()}-${String(today.getMonth() + 1).padStart(2, '0')}-${String(today.getDate()).padStart(2, '0')}` + await createTestSpecimen(ctx.db, specType.id, { collectionDate: todayIso }) + await createTestSpecimen(ctx.db, specType.id, { collectionDate: '2020-01-01' }) + + expect(await report('specimensWithFutureCollectionDate')).toEqual([ + { id: future.id, collectionDate: '2073-02-08' }, + { id: next.id, collectionDate: tomorrowIso }, + ]) + }) + it('lists Containers whose stored position fails the Grid position rule, and no valid or missing ones', async () => { const storageType = await createTestStorageType(ctx.db, { name: 'ShelfBadPos', description: '' }) const loc = await createTestLocation(ctx.db, { name: 'LocBadPos', storageTypeId: String(storageType.id), canContainCollections: true }) diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index 1ff789c9..d7322c3c 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -28,6 +28,13 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Derivation broken references.** container_derivation rows with missing parent or child container. - **Storage container tag orphans.** Tag links that reference a deleted container or tag. - **Duplicate tube barcodes.** Micronix and cryovial tubes that share a barcode, comparing without case or surrounding spaces, so `CK3-3FR4` and `ck3-3fr4` count as one barcode. **Spellings** lists each stored form when they differ. Moves and derivations refuse to pick between these tubes, so identify them by container id or position. +- **Duplicate Specimens.** Specimens of one Subject with the same Specimen type and collection date, dated or undated. Registration treats these as one collection event and adds new Containers to the existing Specimen, but legacy data has groups the lab chose to keep. Fresh installs refuse a second dated Specimen; upgraded databases do not. +- **Subject names that differ only by case.** Subjects in one study whose names are the same ignoring case, such as `3d7_10k` and `3D7_10K`. Subject names in a study ignore case, so the app refuses new ones. Merge or rename the Subjects. +- **Collection names that differ only by case or spaces.** Plates, boxes, or bags of one type, or sheets in one box or bag, whose names are the same ignoring case and surrounding spaces. Names are quoted to show the spaces. Rename one, or delete it if it is empty. +- **Containers in a type their Specimen type does not allow.** Containers whose Container type is not one of the allowed Container types of their Specimen type. Allow the type for the Specimen type in Reference data, or correct the Container. +- **Exhausted Containers with remaining quantity.** Containers whose comment says `exhausted` but whose remaining quantity is above 0, so they show as In Use. Set the remaining quantity to 0 if the material is used up. +- **Placeholder collection dates.** Specimens dated `1970-01-01`, `1999-09-09`, `1899-12-30` or `1111-01-01`, dates the old application stored for an unknown date. Clear the date or enter the real one. An upgrade to schema version 13 clears the legacy ones. +- **Collection dates in the future.** Specimens whose collection date is after today. New dates in the future are refused; correct the legacy ones by hand. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. - **Collection date not ISO.** Specimens whose collection date is not a real date written `YYYY-MM-DD`. Date filters, exports, and study summaries compare collection dates as text, so these Specimens are missed or placed in the wrong range. Correct the date in the database. - **Containers with an invalid Grid position.** Tubes or wells whose stored position is not a well or slot of their plate (A01–H12) or box (A01–J10), or is not stored as a row letter plus a two-digit column (`A1` instead of `A01`). Such a tube shows as unplaced, and another tube can be put in its cell. These count toward the overview issue total. Move each one to its real position, or clear the position. An upgrade to schema version 10 repairs the legacy cases automatically: it strips leading dashes (`--A01` becomes `A01`) and clears positions that are the tube's own barcode or `NA`. diff --git a/packages/web/src/lib/api/admin.ts b/packages/web/src/lib/api/admin.ts index 420617b7..2d04a651 100644 --- a/packages/web/src/lib/api/admin.ts +++ b/packages/web/src/lib/api/admin.ts @@ -104,6 +104,51 @@ export interface ContainerWithInvalidGridPositionItem { position: string } +/** Subject Specimens sharing a Subject, Specimen type and collection date (CONTEXT.md, Specimen identity). */ +export interface DuplicateSpecimenIdentityItem { + studyId: number + studyShortCode: string + subjectId: number + subjectName: string + specimenTypeId: number + specimenTypeName: string + /** Absent for undated Specimens. */ + collectionDate?: string + specimenIds: number[] +} + +export interface CaseVariantSubjectNameItem { + studyId: number + studyShortCode: string + subjects: Array<{ id: number; name: string }> +} + +export interface CaseVariantCollectionNameItem { + type: CollectionType | 'sheet' + /** The box or bag holding the sheets; absent for other Collection types. */ + parent?: { type: 'box' | 'bag'; id: number } + collections: Array<{ id: number; name: string }> +} + +export interface ContainerInDisallowedContainerTypeItem { + id: number + containerType: 'micronix_tube' | 'cryovial_tube' | 'paper' | 'static_well' + specimenId: number + specimenTypeId: number + specimenTypeName: string +} + +export interface ExhaustedContainerWithRemainingQuantityItem { + id: number + comment: string + remainingQuantity: number +} + +export interface SpecimenCollectionDateItem { + id: number + collectionDate: string +} + export interface IntegrityReport { emptyCollections: EmptyCollectionItem[] collectionsWithMissingLocation: CollectionWithMissingLocationItem[] @@ -119,6 +164,13 @@ export interface IntegrityReport { containersWithNoGridPosition: ContainerWithNoGridPositionItem[] specimensWithNonIsoCollectionDate: SpecimenWithNonIsoCollectionDateItem[] containersWithInvalidGridPosition: ContainerWithInvalidGridPositionItem[] + duplicateSpecimenIdentities: DuplicateSpecimenIdentityItem[] + caseVariantSubjectNames: CaseVariantSubjectNameItem[] + caseVariantCollectionNames: CaseVariantCollectionNameItem[] + containersInDisallowedContainerType: ContainerInDisallowedContainerTypeItem[] + exhaustedContainersWithRemainingQuantity: ExhaustedContainerWithRemainingQuantityItem[] + specimensWithPlaceholderCollectionDate: SpecimenCollectionDateItem[] + specimensWithFutureCollectionDate: SpecimenCollectionDateItem[] } export interface CollectionWithMissingLocationItem { diff --git a/packages/web/src/pages/AdminDataIntegrityOverview.tsx b/packages/web/src/pages/AdminDataIntegrityOverview.tsx index 4b676e35..583921cf 100644 --- a/packages/web/src/pages/AdminDataIntegrityOverview.tsx +++ b/packages/web/src/pages/AdminDataIntegrityOverview.tsx @@ -41,7 +41,14 @@ export default function AdminDataIntegrityOverview() { report.duplicateBarcodes.length + report.locationPathInconsistencies.length + report.specimensWithNonIsoCollectionDate.length + - report.containersWithInvalidGridPosition.length + report.containersWithInvalidGridPosition.length + + report.duplicateSpecimenIdentities.length + + report.caseVariantSubjectNames.length + + report.caseVariantCollectionNames.length + + report.containersInDisallowedContainerType.length + + report.exhaustedContainersWithRemainingQuantity.length + + report.specimensWithPlaceholderCollectionDate.length + + report.specimensWithFutureCollectionDate.length // containersWithNoGridPosition is informational and is not an integrity issue count return ( @@ -73,7 +80,7 @@ export default function AdminDataIntegrityOverview() {

{integrityIssueCount}

- Orphans, broken references, duplicate micronix barcodes, path inconsistencies, invalid Grid positions. + Orphans, broken references, duplicate tube barcodes, duplicate Specimens, case-variant names, legacy dates and quantities, invalid Grid positions.

diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index 637adb36..2466c307 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -4,8 +4,10 @@ import { useAdminIntegrityReport } from '../hooks/useAdmin' import { PageError, fromQuery, getQueryErrorMessage } from '../ui' import '../styles/admin.css' -function typeLabel(type: EmptyCollectionItem['type']): string { +function typeLabel(type: EmptyCollectionItem['type'] | 'sheet'): string { switch (type) { + case 'sheet': + return 'Sheet' case 'micronix_plate': return 'Micronix plate' case 'cryovial_box': @@ -19,8 +21,10 @@ function typeLabel(type: EmptyCollectionItem['type']): string { } } -function getCollectionDetailUrl(type: EmptyCollectionItem['type'], id: number): string { +function getCollectionDetailUrl(type: EmptyCollectionItem['type'] | 'sheet', id: number): string { switch (type) { + case 'sheet': + return `/collections/sheets/${id}` case 'micronix_plate': return `/collections/micronix-plates/${id}` case 'cryovial_box': @@ -357,6 +361,235 @@ export default function AdminDataIntegrityReport() { + +
+ + + + + + + + + + + + {report.duplicateSpecimenIdentities.map((g) => ( + + + + + + + + ))} + +
StudySubjectSpecimen typeCollection dateSpecimen IDs
{g.studyShortCode} + + {g.subjectName} + + {g.specimenTypeName}{g.collectionDate ?? 'No date'} + {g.specimenIds.map((id) => ( + + {id} + + ))} +
+
+ + + +
+ + + + + + + + + {report.caseVariantSubjectNames.map((g) => ( + subject.id).join('-')} className="hover:bg-app-surface"> + + + + ))} + +
StudySubjects
{g.studyShortCode} + {g.subjects.map((subject) => ( + + {subject.name} + + ))} +
+
+ + + +
+ + + + + + + + + {report.caseVariantCollectionNames.map((g) => ( + c.id).join('-')}`} className="hover:bg-app-surface"> + + + + ))} + +
TypeCollections
{typeLabel(g.type)} + {g.collections.map((c) => ( + + {JSON.stringify(c.name)} + + ))} +
+
+ + + +
+ + + + + + + + + + + {report.containersInDisallowedContainerType.map((row) => ( + + + + + + + ))} + +
Container IDContainer typeSpecimenSpecimen type
+ + {row.id} + + {row.containerType} + + {row.specimenId} + + {row.specimenTypeName}
+
+ + + +
+ + + + + + + + + + {report.exhaustedContainersWithRemainingQuantity.map((row) => ( + + + + + + ))} + +
Container IDCommentRemaining quantity
+ + {row.id} + + {row.comment}{row.remainingQuantity}
+
+ + + +
+ + + + + + + + + {report.specimensWithPlaceholderCollectionDate.map((s) => ( + + + + + ))} + +
Specimen IDCollection date
+ + {s.id} + + {s.collectionDate}
+
+ + + +
+ + + + + + + + + {report.specimensWithFutureCollectionDate.map((s) => ( + + + + + ))} + +
Specimen IDCollection date
+ + {s.id} + + {s.collectionDate}
+
+ + { }) }) + it('counts every legacy data check as integrity issues', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + duplicateSpecimenIdentities: [ + { studyId: 1, studyShortCode: 'S', subjectId: 2, subjectName: 'A', specimenTypeId: 3, specimenTypeName: 'DBS', specimenIds: [4, 5] }, + ], + caseVariantSubjectNames: [{ studyId: 1, studyShortCode: 'CTRL', subjects: [{ id: 1, name: 'a' }, { id: 2, name: 'A' }] }], + caseVariantCollectionNames: [{ type: 'micronix_plate', collections: [{ id: 1, name: 'P' }, { id: 2, name: 'P ' }] }], + containersInDisallowedContainerType: [ + { id: 1, containerType: 'micronix_tube', specimenId: 2, specimenTypeId: 3, specimenTypeName: 'Polyclonal Ab' }, + ], + exhaustedContainersWithRemainingQuantity: [{ id: 1, comment: 'exhausted', remainingQuantity: 1 }], + specimensWithPlaceholderCollectionDate: [{ id: 1, collectionDate: '1970-01-01' }], + specimensWithFutureCollectionDate: [{ id: 2, collectionDate: '2073-02-08' }], + }) + await render() + await waitFor(() => { + expect(screen.getByRole('link', { name: /integrity report/i })).toHaveTextContent('7') + }) + expect(screen.getByRole('link', { name: /integrity report/i })).toHaveTextContent(/duplicate tube barcodes/i) + }) + it('shows retry when report load fails', async () => { vi.mocked(adminApi.getIntegrityReport).mockRejectedValue(new Error('fail')) await render() diff --git a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx index 49a09902..0c82fa31 100644 --- a/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx +++ b/packages/web/src/pages/__tests__/AdminDataIntegrityReport.test.tsx @@ -19,6 +19,13 @@ function emptyIntegrityReport() { containersWithNoGridPosition: [], specimensWithNonIsoCollectionDate: [], containersWithInvalidGridPosition: [], + duplicateSpecimenIdentities: [], + caseVariantSubjectNames: [], + caseVariantCollectionNames: [], + containersInDisallowedContainerType: [], + exhaustedContainersWithRemainingQuantity: [], + specimensWithPlaceholderCollectionDate: [], + specimensWithFutureCollectionDate: [], } } @@ -44,6 +51,86 @@ describe('AdminDataIntegrityReport', () => { }) }) + it('shows Specimens sharing a Subject, Specimen type and collection date, undated ones as no date', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + duplicateSpecimenIdentities: [ + { studyId: 1, studyShortCode: 'HRP', subjectId: 2, subjectName: 'H-1', specimenTypeId: 3, specimenTypeName: 'DBS', collectionDate: '2022-03-17', specimenIds: [127776, 143389] }, + { studyId: 1, studyShortCode: 'HRP', subjectId: 5, subjectName: 'H-2', specimenTypeId: 3, specimenTypeName: 'DBS', specimenIds: [10, 11] }, + ], + }) + await render() + expect(await screen.findByText('Duplicate Specimens')).toBeInTheDocument() + const rows = screen.getAllByRole('row') + const dated = rows.find((row) => within(row).queryByText('2022-03-17'))! + expect(within(dated).getByRole('link', { name: 'H-1' })).toHaveAttribute('href', '/subjects/2') + expect(within(dated).getByRole('link', { name: '127776' })).toHaveAttribute('href', '/specimens/127776') + expect(within(dated).getByRole('link', { name: '143389' })).toHaveAttribute('href', '/specimens/143389') + const undated = rows.find((row) => within(row).queryByText('H-2'))! + expect(within(undated).getByText('No date')).toBeInTheDocument() + }) + + it('shows Subjects whose names differ only by case', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + caseVariantSubjectNames: [{ studyId: 3, studyShortCode: 'CTRL', subjects: [{ id: 31049, name: '3d7_10k' }, { id: 66543, name: '3D7_10K' }] }], + }) + await render() + expect(await screen.findByText('Subject names that differ only by case')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '3d7_10k' })).toHaveAttribute('href', '/subjects/31049') + expect(screen.getByRole('link', { name: '3D7_10K' })).toHaveAttribute('href', '/subjects/66543') + }) + + it('shows Collections whose names differ only by case or spaces, quoting each name', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + caseVariantCollectionNames: [{ type: 'micronix_plate', collections: [{ id: 1760, name: 'EXVIVO-27 ' }, { id: 1761, name: 'EXVIVO-27' }] }], + }) + await render() + expect(await screen.findByText('Collection names that differ only by case or spaces')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '"EXVIVO-27 "' })).toHaveAttribute('href', '/collections/micronix-plates/1760') + expect(screen.getByRole('link', { name: '"EXVIVO-27"' })).toHaveAttribute('href', '/collections/micronix-plates/1761') + }) + + it('shows Containers in a Container type their Specimen type does not allow', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + containersInDisallowedContainerType: [ + { id: 153439, containerType: 'micronix_tube', specimenId: 141173, specimenTypeId: 47, specimenTypeName: 'Polyclonal Ab' }, + ], + }) + await render() + expect(await screen.findByText('Containers in a type their Specimen type does not allow')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '153439' })).toHaveAttribute('href', '/containers/153439') + expect(screen.getByText('Polyclonal Ab')).toBeInTheDocument() + }) + + it('shows exhausted Containers that still have remaining quantity', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + exhaustedContainersWithRemainingQuantity: [{ id: 50465, comment: 'Z20329; exhausted', remainingQuantity: 1 }], + }) + await render() + expect(await screen.findByText('Exhausted Containers with remaining quantity')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '50465' })).toHaveAttribute('href', '/containers/50465') + expect(screen.getByText('Z20329; exhausted')).toBeInTheDocument() + }) + + it('shows Specimens on a placeholder date and Specimens dated in the future', async () => { + vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ + ...emptyIntegrityReport(), + specimensWithPlaceholderCollectionDate: [{ id: 7, collectionDate: '1970-01-01' }], + specimensWithFutureCollectionDate: [{ id: 8, collectionDate: '2073-02-08' }], + }) + await render() + expect(await screen.findByText('Placeholder collection dates')).toBeInTheDocument() + expect(screen.getByText('Collection dates in the future')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '7' })).toHaveAttribute('href', '/specimens/7') + expect(screen.getByText('1970-01-01')).toBeInTheDocument() + expect(screen.getByRole('link', { name: '8' })).toHaveAttribute('href', '/specimens/8') + expect(screen.getByText('2073-02-08')).toBeInTheDocument() + }) + it('shows containers with no grid position when present', async () => { vi.mocked(adminApi.getIntegrityReport).mockResolvedValue({ ...emptyIntegrityReport(), From c671fc32336b135ce0cb24c5e564df0355a75936 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:30:38 -0800 Subject: [PATCH 48/54] fix(migrations): 008 and 010 leave what they can't fix instead of aborting Migrations run at startup on every install, so an install with a value their rule can't handle could not start until someone fixed it in SQL. - 008 converts M/D/YY and M/D/YYYY collection dates that are real calendar dates, trims whitespace and nulls blanks as before, and leaves every other value exactly as it was. - 010 applies a repair only when no other Container holds, or another repair would take, the same position in that Collection; every repair to a contested position is left, so none is preferred. A position that fits no repair case is left as it was. - The integrity report's existing "Collection date not ISO" and "invalid Grid position" checks list what is left; the tests run them on the migrated database. Docs and CONTEXT.md say what is left and why. On the 2026-10-31 production snapshot the result is identical to the aborting versions: 1,148 dates converted, 160 positions normalized and 206 cleared, nothing left over. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 2 +- .../src/db/__tests__/schema-evolution.test.ts | 104 +++++++++++------- .../db/migrations/008_collection_date_iso.sql | 19 +--- .../migrations/010_repair_grid_positions.sql | 37 +++---- .../docs/guides/advanced/data-integrity.md | 4 +- .../src/pages/AdminDataIntegrityReport.tsx | 2 +- 6 files changed, 87 insertions(+), 81 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 3ec656f3..c5e92cfd 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -164,7 +164,7 @@ Where a **Container** sits in the physical storage hierarchy — its **grid posi _Avoid_: Enrichment (implementation term); conflating placement with specimen or source lookups; placing paper directly on a box or bag without a sheet **Grid position**: -The well or slot of a tube or well **Container** in its plate or box, stored as a row letter plus a two-digit column (A01). One rule, the Grid position rule, says which positions a **Collection** has (micronix plate A01–H12, cryovial box A01–J10) and how a typed position maps onto one: `a1`, `A1` and `a01` all mean A01, and anything else, including a cell outside the plate or box, is not a Grid position. Every write that sets a position applies the rule and rejects a position it does not accept; every read that lays Containers out on the grid applies the same rule, so a position a write would reject places no Container (see **Unplaced container**). Legacy malformed positions were repaired by migration 010, and the integrity report flags any stored position that fails the rule. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. +The well or slot of a tube or well **Container** in its plate or box, stored as a row letter plus a two-digit column (A01). One rule, the Grid position rule, says which positions a **Collection** has (micronix plate A01–H12, cryovial box A01–J10) and how a typed position maps onto one: `a1`, `A1` and `a01` all mean A01, and anything else, including a cell outside the plate or box, is not a Grid position. Every write that sets a position applies the rule and rejects a position it does not accept; every read that lays Containers out on the grid applies the same rule, so a position a write would reject places no Container (see **Unplaced container**). Migration 010 repairs the legacy malformed positions it can and leaves the rest as they were; the integrity report flags any stored position that fails the rule. Optional: a tube or well may belong to a **Collection** with no grid position (legacy rows). At most one Container occupies a given grid position in a Collection; a missing position occupies no cell. Not applicable to **Paper**. _Avoid_: Well (as the domain term); treating a missing position as occupying a slot; requiring every tube to have a position before it can exist in a Collection **Unplaced container**: diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index f3d3da92..0cf32f2f 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -12,6 +12,9 @@ import { } from '../schema-evolution' import { openOperationalDatabase } from '../open' import { SchemaMigrationError, listNumberedMigrations, runSqlMigration, runSqlMigrationStatements } from '../migration-runner' +import { drizzle } from 'drizzle-orm/bun-sqlite' +import * as schema from '../schema' +import { listContainersWithInvalidGridPosition, listSpecimensWithNonIsoCollectionDate } from '../../lib/data-audit' /** Tables that migrations 004+ alter, so older-version fixtures can migrate to current. */ function createTablesForLaterMigrations(sqlite: Database): void { @@ -371,23 +374,28 @@ describe('schema evolution', () => { sqlite.close() }) - it.each(['13/2/2024', '2/30/2024', '2024-02-30', '5/7', '7 Apr 2024', '2024/05/07', '5/7/124'])( - 'migration 008 aborts on a collection date it cannot read (%s) and changes nothing', - (bad) => { - const sqlite = new Database(testDbPath) - sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') - sqlite.exec('INSERT INTO schema_version (version) VALUES (7)') - createTablesForLaterMigrations(sqlite) - sqlite.exec(`INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (1, 1, '5/7/26')`) - sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (2, 1, ?)').run(bad) - - expect(() => evolveOperationalSchema(sqlite)).toThrow(/Cannot convert collection dates to YYYY-MM-DD/) - expect(getRecordedSchemaVersion(sqlite)).toBe(7) - const rows = sqlite.prepare('SELECT collection_date AS date FROM specimen ORDER BY id').all() as Array<{ date: string }> - expect(rows.map((r) => r.date)).toEqual(['5/7/26', bad]) - sqlite.close() - }, - ) + it('migration 008 leaves a collection date it cannot read as it is, converts the rest, and the integrity report lists it', async () => { + const sqlite = new Database(testDbPath) + sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') + sqlite.exec('INSERT INTO schema_version (version) VALUES (7)') + createTablesForLaterMigrations(sqlite) + const unreadable = ['13/2/2024', '2/30/2024', '2024-02-30', '5/7', '7 Apr 2024', '2024/05/07', '5/7/124', ' 2/30/24 '] + const insert = sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (?, 1, ?)') + insert.run(1, '5/7/26') + unreadable.forEach((value, i) => insert.run(i + 2, value)) + insert.run(100, ' 12/24/2024 ') + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + const rows = sqlite.prepare('SELECT collection_date AS date FROM specimen ORDER BY id').all() as Array<{ date: string }> + expect(rows.map((r) => r.date)).toEqual(['2026-05-07', ...unreadable, '2024-12-24']) + const temp = sqlite.prepare(`SELECT name FROM sqlite_temp_master WHERE name LIKE '__migration_008%'`).all() + expect(temp).toEqual([]) + const leftovers = await listSpecimensWithNonIsoCollectionDate(drizzle(sqlite, { schema })) + expect(leftovers).toEqual(unreadable.map((collectionDate, i) => ({ id: i + 2, collectionDate }))) + sqlite.close() + }) /** The CREATE TABLE text SQLite stored for qpcr_well_result. */ function qpcrWellResultDdl(sqlite: Database): string { @@ -500,23 +508,27 @@ describe('schema evolution', () => { sqlite.close() }) - it('migration 010 aborts, changing nothing, when a stripped position collides with a stored one', () => { + it('migration 010 leaves a repair that would collide with a stored position as it is, and repairs the rest', () => { const sqlite = databaseBeforeMigration010() sqlite.exec(`INSERT INTO cryovial_tube (id, collection_id, barcode, position) VALUES (1, 790, 'C1', '--A01'), (2, 790, 'C2', 'A01'), (3, 790, 'C3', '-B01')`) - expect(() => evolveOperationalSchema(sqlite)).toThrow(/would put two Containers in one position of the same Collection/) - expect(getRecordedSchemaVersion(sqlite)).toBe(9) - expect(positions(sqlite, 'cryovial_tube')).toEqual([[1, '--A01'], [2, 'A01'], [3, '-B01']]) + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(positions(sqlite, 'cryovial_tube')).toEqual([[1, '--A01'], [2, 'A01'], [3, 'B01']]) sqlite.close() }) - it('migration 010 aborts when two repairs land in one position', () => { + it('migration 010 leaves every repair that lands in one position as it is, and repairs the rest', () => { const sqlite = databaseBeforeMigration010() - sqlite.exec(`INSERT INTO micronix_tube (id, collection_id, barcode, position) VALUES (1, 5, 'M1', '--C04')`) + sqlite.exec(`INSERT INTO micronix_tube (id, collection_id, barcode, position) VALUES (1, 5, 'M1', '--C04'), (3, 5, 'M3', '-d5')`) sqlite.exec(`INSERT INTO static_well (id, collection_id, position) VALUES (2, 5, 'c4')`) - expect(() => evolveOperationalSchema(sqlite)).toThrow(/would put two Containers in one position/) - expect(getRecordedSchemaVersion(sqlite)).toBe(9) + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(positions(sqlite, 'micronix_tube')).toEqual([[1, '--C04'], [3, 'D05']]) + expect(positions(sqlite, 'static_well')).toEqual([[2, 'c4']]) sqlite.close() }) @@ -530,23 +542,31 @@ describe('schema evolution', () => { sqlite.close() }) - for (const [table, position] of [ - ['cryovial_tube', 'K01'], - ['cryovial_tube', '--A11'], - ['micronix_tube', 'I01'], - ['micronix_tube', 'A001'], - ['micronix_tube', 'hello'], - ] as const) { - it(`migration 010 aborts on a position it cannot repair: ${table} "${position}"`, () => { - const sqlite = databaseBeforeMigration010() - sqlite.exec(`INSERT INTO ${table} (id, collection_id, barcode, position) VALUES (1, 1, 'X1', '--A02'), (2, 1, 'X2', '${position}')`) - - expect(() => evolveOperationalSchema(sqlite)).toThrow(/cannot be repaired automatically/) - expect(getRecordedSchemaVersion(sqlite)).toBe(9) - expect(positions(sqlite, table)).toEqual([[1, '--A02'], [2, position]]) - sqlite.close() - }) - } + it('migration 010 leaves a position it cannot repair as it is, repairs the rest, and the integrity report lists what is left', async () => { + const sqlite = databaseBeforeMigration010() + sqlite.exec(`INSERT INTO cryovial_tube (id, collection_id, barcode, position) VALUES + (1, 1, 'X1', '--A02'), (2, 1, 'X2', 'K01'), (3, 1, 'X3', '--A11'), (4, 1, 'X4', ' NA01 '), (5, 1, 'X5', '--A03'), (6, 1, 'X6', 'A03')`) + sqlite.exec(`INSERT INTO micronix_tube (id, collection_id, barcode, position) VALUES + (11, 2, 'M1', 'I01'), (12, 2, 'M2', 'A001'), (13, 2, 'M3', 'hello'), (14, 2, 'M4', 'b7')`) + + evolveOperationalSchema(sqlite) + + expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) + expect(positions(sqlite, 'cryovial_tube')).toEqual([[1, 'A02'], [2, 'K01'], [3, '--A11'], [4, null], [5, '--A03'], [6, 'A03']]) + expect(positions(sqlite, 'micronix_tube')).toEqual([[11, 'I01'], [12, 'A001'], [13, 'hello'], [14, 'B07']]) + const temp = sqlite.prepare(`SELECT name FROM sqlite_temp_master WHERE name LIKE '__migration_010%'`).all() + expect(temp).toEqual([]) + const leftovers = await listContainersWithInvalidGridPosition(drizzle(sqlite, { schema })) + expect(leftovers).toEqual([ + { id: 11, containerType: 'micronix_tube', collectionId: 2, position: 'I01' }, + { id: 12, containerType: 'micronix_tube', collectionId: 2, position: 'A001' }, + { id: 13, containerType: 'micronix_tube', collectionId: 2, position: 'hello' }, + { id: 2, containerType: 'cryovial_tube', collectionId: 1, position: 'K01' }, + { id: 3, containerType: 'cryovial_tube', collectionId: 1, position: '--A11' }, + { id: 5, containerType: 'cryovial_tube', collectionId: 1, position: '--A03' }, + ]) + sqlite.close() + }) /** A database at version 11 with control batches, ready for migration 012. */ function databaseBeforeMigration012(): Database { diff --git a/packages/api/src/db/migrations/008_collection_date_iso.sql b/packages/api/src/db/migrations/008_collection_date_iso.sql index a8e9e2e9..71a6936a 100644 --- a/packages/api/src/db/migrations/008_collection_date_iso.sql +++ b/packages/api/src/db/migrations/008_collection_date_iso.sql @@ -1,7 +1,9 @@ --- Migration 008: every Specimen collection date is YYYY-MM-DD (issue #104). +-- Migration 008: store Specimen collection dates as YYYY-MM-DD (issue #104). -- Rewrites legacy M/D/YY and M/D/YYYY values (US month first; a 2-digit year is 20YY) and --- surrounding whitespace, and turns blank values into NULL. Abort if any other value remains, --- so it can be reviewed by hand. +-- surrounding whitespace, and turns blank values into NULL. A value in any other format, or one +-- that is not a real calendar date, is left exactly as it is: the migration runs at startup on +-- every install, so it must not stop one. The integrity report's "Collection date not ISO" +-- check lists what is left, for an admin to correct. UPDATE specimen SET collection_date = NULL WHERE typeof(collection_date) = 'text' AND trim(collection_date) = '';--> statement-breakpoint CREATE TEMP TABLE __migration_008_dates AS @@ -37,16 +39,7 @@ candidates AS ( FROM fields ) SELECT id, CASE WHEN date(iso) IS iso THEN iso END AS iso FROM candidates;--> statement-breakpoint -CREATE TEMP TABLE IF NOT EXISTS __migration_008_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint -CREATE TRIGGER __migration_008_guard_tr BEFORE INSERT ON __migration_008_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot convert collection dates to YYYY-MM-DD: a specimen.collection_date is not YYYY-MM-DD, M/D/YY or M/D/YYYY, or is not a real calendar date. Fix it by hand first. SELECT id, collection_date FROM specimen WHERE date(collection_date) IS NOT collection_date lists every value that is not YYYY-MM-DD.') - WHERE EXISTS (SELECT 1 FROM __migration_008_dates WHERE iso IS NULL); -END;--> statement-breakpoint -INSERT INTO __migration_008_guard DEFAULT VALUES;--> statement-breakpoint -DROP TRIGGER __migration_008_guard_tr;--> statement-breakpoint -DROP TABLE __migration_008_guard;--> statement-breakpoint UPDATE specimen SET collection_date = (SELECT iso FROM __migration_008_dates WHERE __migration_008_dates.id = specimen.id) -WHERE id IN (SELECT id FROM __migration_008_dates);--> statement-breakpoint +WHERE id IN (SELECT id FROM __migration_008_dates WHERE iso IS NOT NULL);--> statement-breakpoint DROP TABLE __migration_008_dates; diff --git a/packages/api/src/db/migrations/010_repair_grid_positions.sql b/packages/api/src/db/migrations/010_repair_grid_positions.sql index 53c7f346..6a97f433 100644 --- a/packages/api/src/db/migrations/010_repair_grid_positions.sql +++ b/packages/api/src/db/migrations/010_repair_grid_positions.sql @@ -1,4 +1,4 @@ --- Migration 010: every stored Grid position passes the Grid position rule (issue #125). +-- Migration 010: repair stored Grid positions to pass the Grid position rule (issue #125). -- The rule is `parseGridPosition` in packages/contract/src/grid-position.ts: a row letter and -- a one- or two-digit column inside the Collection's geometry (micronix plate A01–H12, cryovial -- box A01–J10), stored as an upper-case letter and a two-digit column. For each position not @@ -42,34 +42,27 @@ SELECT tbl, id, position AS old_position, grid AS new_position, END AS repair FROM parsed WHERE grid IS NULL OR grid <> position;--> statement-breakpoint +DELETE FROM __migration_010_repair WHERE repair = 'unexpected';--> statement-breakpoint CREATE TEMP TABLE __migration_010_final AS -SELECT t.grid_type, t.collection_id, - CASE WHEN r.id IS NULL THEN t.position ELSE r.new_position END AS position, - r.id IS NOT NULL AS repaired +SELECT t.tbl, t.id, t.grid_type, t.collection_id, + CASE WHEN r.id IS NULL THEN t.position ELSE r.new_position END AS position FROM ( SELECT 'micronix_tube' AS tbl, 'micronix_plate' AS grid_type, id, collection_id, position FROM micronix_tube UNION ALL SELECT 'cryovial_tube', 'cryovial_box', id, collection_id, position FROM cryovial_tube UNION ALL SELECT 'static_well', 'micronix_plate', id, collection_id, position FROM static_well ) t LEFT JOIN __migration_010_repair r ON r.tbl = t.tbl AND r.id = t.id;--> statement-breakpoint -CREATE TEMP TABLE IF NOT EXISTS __migration_010_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint -CREATE TRIGGER __migration_010_unexpected_tr BEFORE INSERT ON __migration_010_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot repair Grid positions: a stored position is not dash-prefixed, the tube''s barcode or NA, so it cannot be repaired automatically. Correct or clear it by hand first.') - WHERE EXISTS (SELECT 1 FROM __migration_010_repair WHERE repair = 'unexpected'); -END;--> statement-breakpoint -CREATE TRIGGER __migration_010_collision_tr BEFORE INSERT ON __migration_010_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot repair Grid positions: a repaired position would put two Containers in one position of the same Collection. Move or clear one of them first.') - WHERE EXISTS ( - SELECT 1 FROM __migration_010_final WHERE position IS NOT NULL - GROUP BY grid_type, collection_id, position HAVING COUNT(*) > 1 AND MAX(repaired) = 1 - ); -END;--> statement-breakpoint -INSERT INTO __migration_010_guard DEFAULT VALUES;--> statement-breakpoint -DROP TRIGGER __migration_010_unexpected_tr;--> statement-breakpoint -DROP TRIGGER __migration_010_collision_tr;--> statement-breakpoint -DROP TABLE __migration_010_guard;--> statement-breakpoint +DELETE FROM __migration_010_repair +WHERE new_position IS NOT NULL + AND EXISTS ( + SELECT 1 FROM __migration_010_final self + JOIN __migration_010_final other + ON other.grid_type = self.grid_type + AND other.collection_id = self.collection_id + AND other.position = self.position + AND NOT (other.tbl = self.tbl AND other.id = self.id) + WHERE self.tbl = __migration_010_repair.tbl AND self.id = __migration_010_repair.id + );--> statement-breakpoint UPDATE micronix_tube SET position = (SELECT r.new_position FROM __migration_010_repair r WHERE r.tbl = 'micronix_tube' AND r.id = micronix_tube.id) WHERE id IN (SELECT id FROM __migration_010_repair WHERE tbl = 'micronix_tube');--> statement-breakpoint diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index 1ff789c9..6a96de8d 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -29,8 +29,8 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Storage container tag orphans.** Tag links that reference a deleted container or tag. - **Duplicate tube barcodes.** Micronix and cryovial tubes that share a barcode, comparing without case or surrounding spaces, so `CK3-3FR4` and `ck3-3fr4` count as one barcode. **Spellings** lists each stored form when they differ. Moves and derivations refuse to pick between these tubes, so identify them by container id or position. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. -- **Collection date not ISO.** Specimens whose collection date is not a real date written `YYYY-MM-DD`. Date filters, exports, and study summaries compare collection dates as text, so these Specimens are missed or placed in the wrong range. Correct the date in the database. -- **Containers with an invalid Grid position.** Tubes or wells whose stored position is not a well or slot of their plate (A01–H12) or box (A01–J10), or is not stored as a row letter plus a two-digit column (`A1` instead of `A01`). Such a tube shows as unplaced, and another tube can be put in its cell. These count toward the overview issue total. Move each one to its real position, or clear the position. An upgrade to schema version 10 repairs the legacy cases automatically: it strips leading dashes (`--A01` becomes `A01`) and clears positions that are the tube's own barcode or `NA`. +- **Collection date not ISO.** Specimens whose collection date is not a real date written `YYYY-MM-DD`. Date filters, exports, and study summaries can't read such a date, so they leave these Specimens out. An upgrade to schema version 8 converts `M/D/YY` and `M/D/YYYY` dates and leaves any other value as it was, for you to find here. Correct the date of each Specimen listed. +- **Containers with an invalid Grid position.** Tubes or wells whose stored position is not a well or slot of their plate (A01–H12) or box (A01–J10), or is not stored as a row letter plus a two-digit column (`A1` instead of `A01`). Such a tube shows as unplaced, and another tube can be put in its cell. These count toward the overview issue total. Move each one to its real position, or clear the position. An upgrade to schema version 10 repairs the legacy cases automatically: it strips leading dashes (`--A01` becomes `A01`) and clears positions that are the tube's own barcode or `NA`. It leaves a position as it was, for you to find here, when it fits none of these cases or when repairing it would put two Containers in one position of a Collection. - **Containers with no grid position.** Tubes or wells in a collection with no well or slot. Legacy rows are allowed and do **not** count toward the overview issue total. Assign a position via edit or scan move when the well is known. ## Empty collections diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index 637adb36..779eab03 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -393,7 +393,7 @@ export default function AdminDataIntegrityReport() {
From 936aee9669a6405a89194e837dd3b5f06ce9626a Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:31:14 -0800 Subject: [PATCH 49/54] fix(dates): keep a collection date migration 008 left out of filters, ranges and sorts Migration 008 now leaves a collection date it can't read (13/2/2024) as it was. Compared as text, such a value passed or failed a date bound by accident and became the earliest or latest date of a range. - Specimen list, statistics and Container selection date filters also require a real YYYY-MM-DD date (isCalendarDate in sql-helpers.ts). - Sorting the specimen list by collection date puts such a value after every real date and before a missing one. - Study summary, study summaries, study timeline, subject and control batch summaries and the monthly collection timeline read only real dates; a subject timeline places such a Specimen by its created time. Co-Authored-By: Claude Opus 5.5 --- packages/api/src/lib/container-enrichment.ts | 8 +- .../src/lib/container-selection/conditions.ts | 11 +- .../api/src/lib/container-selection/index.ts | 4 +- .../api/src/lib/container-selection/scopes.ts | 6 +- packages/api/src/lib/date-filters.ts | 3 +- .../api/src/lib/specimens/specimen-read.ts | 11 +- packages/api/src/lib/sql-helpers.ts | 11 +- .../src/lib/statistics/specimen-aggregates.ts | 4 +- packages/api/src/lib/studies/study-read.ts | 6 +- .../leftover-collection-dates.test.ts | 135 ++++++++++++++++++ packages/api/src/routes/studies.ts | 16 ++- 11 files changed, 193 insertions(+), 22 deletions(-) create mode 100644 packages/api/src/routes/__tests__/leftover-collection-dates.test.ts diff --git a/packages/api/src/lib/container-enrichment.ts b/packages/api/src/lib/container-enrichment.ts index f16b7839..10b64b2f 100644 --- a/packages/api/src/lib/container-enrichment.ts +++ b/packages/api/src/lib/container-enrichment.ts @@ -1,5 +1,6 @@ import type { Database } from '../db/client' import { parseStoredTimestamp } from './datetime' +import { isIsoCalendarDate } from './date-filters' import { resolveContainerPlacementBundle, type ContainerPlacement, @@ -211,10 +212,11 @@ export function buildSpecimenCollectionSummary( containerTypeCounts[placement.containerType] = (containerTypeCounts[placement.containerType] || 0) + 1 } + // Only real YYYY-MM-DD dates: migration 008 leaves one it cannot read (`13/2/2024`) as it was. const collectionDates = enrichedSpecimens .map((s) => s.collectionDate) - .filter(Boolean) - .sort() as string[] + .filter((date): date is string => date != null && isIsoCalendarDate(date)) + .sort() const collectionDateRange = collectionDates.length > 0 ? { earliest: collectionDates[0], latest: collectionDates[collectionDates.length - 1] } : null @@ -222,7 +224,7 @@ export function buildSpecimenCollectionSummary( const timeline: SpecimenTimelineEntry[] = enrichedSpecimens .map((spec) => ({ id: spec.id, - date: spec.collectionDate || spec.created, + date: spec.collectionDate != null && isIsoCalendarDate(spec.collectionDate) ? spec.collectionDate : spec.created, specimenTypeName: spec.specimenTypeName, specimenTypeId: spec.specimenTypeId, })) diff --git a/packages/api/src/lib/container-selection/conditions.ts b/packages/api/src/lib/container-selection/conditions.ts index c92cbfb7..9a8a6a8d 100644 --- a/packages/api/src/lib/container-selection/conditions.ts +++ b/packages/api/src/lib/container-selection/conditions.ts @@ -16,7 +16,7 @@ import { storageContainerTag, } from '../../db/schema' import type { ContainerSubtype } from '../container-placement' -import { dateToUpperBound } from '../sql-helpers' +import { dateToUpperBound, isCalendarDate } from '../sql-helpers' import type { ContainerSelectionFilters, DateRange, SpecimenSourceType } from './types' /** @@ -39,6 +39,15 @@ export function dateRangeConditions(column: SQLiteColumn, range: DateRange | und ] } +/** + * Conditions for an inclusive range on the Specimen collection date. A bound also excludes a + * collection date that is not a real YYYY-MM-DD date (see `isCalendarDate`). + */ +export function collectionDateConditions(range: DateRange | undefined): SQL[] { + const conditions = dateRangeConditions(specimen.collectionDate, range) + return conditions.length > 0 ? [...conditions, isCalendarDate(specimen.collectionDate)] : [] +} + /** Specimens of the given source type; undefined, every Specimen. */ export function sourceTypeCondition(sourceType: SpecimenSourceType | undefined): SQL | undefined { if (sourceType === undefined) return undefined diff --git a/packages/api/src/lib/container-selection/index.ts b/packages/api/src/lib/container-selection/index.ts index a8310a64..39e5b0c7 100644 --- a/packages/api/src/lib/container-selection/index.ts +++ b/packages/api/src/lib/container-selection/index.ts @@ -10,8 +10,8 @@ import { CONTAINER_SUBTYPES, type ContainerSubtype } from '../container-placemen import { collectionDateFilter, timestampFilter } from '../date-filters' import { containerFilterConditions, + collectionDateConditions, containerTypeSql, - dateRangeConditions, sourceTypeCondition, specimenFilterConditions, } from './conditions' @@ -85,7 +85,7 @@ async function selectionScopes( const filters = checkedDateFilters(selection.filters ?? {}) if (selection.kind === 'subjects') checkSubjectDates(selection) - const collectionDate = dateRangeConditions(specimen.collectionDate, filters.collectionDate) + const collectionDate = collectionDateConditions(filters.collectionDate) let scopes: SelectionScope[] switch (selection.kind) { case 'study': { diff --git a/packages/api/src/lib/container-selection/scopes.ts b/packages/api/src/lib/container-selection/scopes.ts index 44fdea1d..dd2d68b8 100644 --- a/packages/api/src/lib/container-selection/scopes.ts +++ b/packages/api/src/lib/container-selection/scopes.ts @@ -4,7 +4,7 @@ import type { Database } from '../../db/client' import { micronixTube, specimen, storageContainer, studySubject } from '../../db/schema' import { shiftCalendarDate } from '../datetime' import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' -import { dateRangeConditions } from './conditions' +import { collectionDateConditions } from './conditions' import type { SubjectSelectionEntry } from './types' /** @@ -97,8 +97,8 @@ function subjectDateCondition(dateFilter: SubjectDateFilter, toleranceDays: numb const from = shiftCalendarDate(dateFilter.exact, -toleranceDays) const to = shiftCalendarDate(dateFilter.exact, toleranceDays) if (!from || !to) return eq(specimen.collectionDate, dateFilter.exact) - return and(...dateRangeConditions(specimen.collectionDate, { from, to })) + return and(...collectionDateConditions({ from, to })) } - const range = dateRangeConditions(specimen.collectionDate, dateFilter) + const range = collectionDateConditions(dateFilter) return range.length > 0 ? and(...range) : undefined } diff --git a/packages/api/src/lib/date-filters.ts b/packages/api/src/lib/date-filters.ts index 4a01776a..6dba1115 100644 --- a/packages/api/src/lib/date-filters.ts +++ b/packages/api/src/lib/date-filters.ts @@ -1,7 +1,8 @@ /** * Date filter bounds from a request (issue #104). Collection dates are stored as YYYY-MM-DD and * compared as strings, so a malformed bound such as `garbage` or `5/7/2024` would silently - * match everything or nothing. These helpers reject it with a `ValidationError` (400) instead. + * match everything or nothing. (A stored date migration 008 could not read stays as it was; + * `isCalendarDate` in `sql-helpers.ts` keeps it out of a filter.) These helpers reject it with a `ValidationError` (400) instead. */ import { ValidationError } from './error-handler' import { parseCollectionDate } from './validation' diff --git a/packages/api/src/lib/specimens/specimen-read.ts b/packages/api/src/lib/specimens/specimen-read.ts index 7fff00a3..f22bb3a3 100644 --- a/packages/api/src/lib/specimens/specimen-read.ts +++ b/packages/api/src/lib/specimens/specimen-read.ts @@ -6,7 +6,7 @@ import { idQueryParam } from '../query-params' import { ValidationError } from '../error-handler' import { findContainersByBarcode } from '../identifier-resolution' import { collectionDateFilter, timestampFilter } from '../date-filters' -import { dateRangeConditions } from '../sql-helpers' +import { dateRangeConditions, isCalendarDate } from '../sql-helpers' import { likeContains } from '../sql-like' export type ListSpecimensQuery = { @@ -62,12 +62,18 @@ function isSpecimenSortColumn(value: string): value is SpecimenSortColumn { /** * ORDER BY for a sort. Specimens without a collection date come last in either direction, + * just after those whose collection date is not a real YYYY-MM-DD date (left by migration 008), * and the id breaks ties so that consecutive pages neither repeat nor skip a row. */ function specimenOrderBy(sort: SpecimenSortColumn, order: SortOrder) { const direction = order === 'asc' ? asc : desc if (sort === 'collection_date') { - return [sql`${specimen.collectionDate} IS NULL`, direction(specimen.collectionDate), direction(specimen.id)] + return [ + sql`${specimen.collectionDate} IS NULL`, + sql`NOT ${isCalendarDate(specimen.collectionDate)}`, + direction(specimen.collectionDate), + direction(specimen.id), + ] } return [direction(specimen.created), direction(specimen.id)] } @@ -201,6 +207,7 @@ export async function listSpecimens(database: Database, query: ListSpecimensQuer if (collectionTo) { conditions.push(sql`${specimen.collectionDate} <= ${collectionTo}`) } + if (collectionFrom || collectionTo) conditions.push(isCalendarDate(specimen.collectionDate)) conditions.push(...dateRangeConditions(specimen.created, createdFromBound, createdToBound)) if (barcode) { diff --git a/packages/api/src/lib/sql-helpers.ts b/packages/api/src/lib/sql-helpers.ts index 55bea307..d828a689 100644 --- a/packages/api/src/lib/sql-helpers.ts +++ b/packages/api/src/lib/sql-helpers.ts @@ -1,4 +1,4 @@ -import { gte, lt, lte, type SQL } from 'drizzle-orm' +import { gte, lt, lte, sql, type SQL } from 'drizzle-orm' import type { SQLiteColumn } from 'drizzle-orm/sqlite-core' import { shiftCalendarDate, toStoredTimestamp } from './datetime' @@ -15,6 +15,15 @@ export function timestampBefore(column: SQLiteColumn, instant: Date): SQL { return lt(column, toStoredTimestamp(instant)) } +/** + * `column` holds a real calendar date written YYYY-MM-DD. Migration 008 leaves a collection date + * it cannot read (`13/2/2024`) as it was, and compared as text such a value passes or fails a + * date bound by accident; add this condition to keep it out of every date filter. + */ +export function isCalendarDate(column: SQLiteColumn): SQL { + return sql`date(${column}) IS ${column}` +} + /** * Upper bound for a "to" date filter. A bare YYYY-MM-DD includes that whole day, so a * timestamp column ('2026-05-26 22:28:26') is compared with "< next day", not "<= date". diff --git a/packages/api/src/lib/statistics/specimen-aggregates.ts b/packages/api/src/lib/statistics/specimen-aggregates.ts index 7407bfba..57576e1e 100644 --- a/packages/api/src/lib/statistics/specimen-aggregates.ts +++ b/packages/api/src/lib/statistics/specimen-aggregates.ts @@ -4,6 +4,7 @@ import { inArray } from 'drizzle-orm' import { cache, cacheKeys } from '../cache' import { chunkArray, SQL_IN_CHUNK_SIZE } from '../sql-helpers' import { utcMonthOf } from '../datetime' +import { isIsoCalendarDate } from '../date-filters' export type SpecimenAggregates = { total: number @@ -26,7 +27,8 @@ function buildMonthlyTimeline( const monthMap = new Map() specimens - .filter((s) => (dateField === 'collectionDate' ? s.collectionDate : true)) + // Only real YYYY-MM-DD collection dates: migration 008 leaves one it cannot read as it was. + .filter((s) => (dateField === 'collectionDate' ? s.collectionDate != null && isIsoCalendarDate(s.collectionDate) : true)) .forEach((s) => { const rawDate = dateField === 'collectionDate' ? s.collectionDate! : s.created // UTC month: `created` is a UTC timestamp and a collection date has no time of day. diff --git a/packages/api/src/lib/studies/study-read.ts b/packages/api/src/lib/studies/study-read.ts index b40cd6fc..01cd070a 100644 --- a/packages/api/src/lib/studies/study-read.ts +++ b/packages/api/src/lib/studies/study-read.ts @@ -4,6 +4,7 @@ import { study, studySubject } from '../../db/schema' import { and, eq, inArray, or, sql } from 'drizzle-orm' import { SQL_IN_CHUNK_SIZE, dateRangeConditions } from '../sql-helpers' import { likeContains } from '../sql-like' +import { isIsoCalendarDate } from '../date-filters' export type ListStudiesParams = { search?: string @@ -194,9 +195,10 @@ export async function getStudySummaries( const totalSpecimens = studySpecimens.length const totalContainers = containerCounts[id] || 0 - const collectionDates = (studySpecimens + // Only real YYYY-MM-DD dates: migration 008 leaves one it cannot read (`13/2/2024`) as it was. + const collectionDates = studySpecimens .map((s) => s.collectionDate) - .filter(Boolean) as string[]) + .filter((date): date is string => date != null && isIsoCalendarDate(date)) .sort() const collectionDateRange = collectionDates.length > 0 diff --git a/packages/api/src/routes/__tests__/leftover-collection-dates.test.ts b/packages/api/src/routes/__tests__/leftover-collection-dates.test.ts new file mode 100644 index 00000000..92369d95 --- /dev/null +++ b/packages/api/src/routes/__tests__/leftover-collection-dates.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { + setupAuthenticatedRouteTest, + type AuthenticatedRouteTestContext, +} from '../../__tests__/helpers/authenticated-route-test' +import { createSpecimensRoutes } from '../specimens' +import { createStatisticsRoutes } from '../statistics' +import { createStudiesRoutes } from '../studies' +import { createTestSpecimen, createTestSpecimenType, createTestStudy, createTestStudySubject } from '../../__tests__/helpers/factories' +import { buildSpecimenCollectionSummary, type EnrichedSpecimenSummary } from '../../lib/container-enrichment' +import { computeSpecimenAggregates } from '../../lib/statistics/specimen-aggregates' + +// Migration 008 leaves a collection date it cannot read as it was (`13/2/2024`, `Apr 2025`). +// Compared as text, `13/2/2024` sorts before every YYYY-MM-DD date and `Apr 2025` after, so +// each would pass or fail a date bound by accident. Date filters, ranges and sorts read only +// real YYYY-MM-DD dates; the integrity report lists the rest. +describe('collection dates migration 008 left as they were', () => { + let ctx: AuthenticatedRouteTestContext + + beforeEach(async () => { + ctx = await setupAuthenticatedRouteTest({ + user: { email: 'admin@test.com', name: 'Admin', password: 'password123', role: 'admin' }, + settings: { pagination: true }, + mount: (app, { db, sqlite }) => { + app.route('/api/specimens', createSpecimensRoutes(db)) + app.route('/api/statistics', createStatisticsRoutes(db)) + app.route('/api/studies', createStudiesRoutes(db, sqlite)) + }, + }) + }) + + afterEach(() => { + ctx.cleanup() + }) + + /** A study with Specimens collected on `dates`, in that order. */ + async function seedStudy(dates: Array) { + const studyRecord = await createTestStudy(ctx.db, { title: 'Legacy dates', shortCode: 'LEG' }) + const subject = await createTestStudySubject(ctx.db, { studyId: studyRecord.id, name: 'S1' }) + const type = await createTestSpecimenType(ctx.db, { name: 'DBS' }) + const specimens = [] + for (const collectionDate of dates) { + specimens.push(await createTestSpecimen(ctx.db, type.id, { studySubjectId: subject.id, collectionDate })) + } + return { studyRecord, type, specimens } + } + + it('the specimen list matches no date bound on a leftover value', async () => { + const { specimens } = await seedStudy(['1999-12-31', '2024-05-07', '13/2/2024', 'Apr 2025']) + + const to = await ctx.request('/api/specimens?collection_date_to=2000-01-01') + const from = await ctx.request('/api/specimens?collection_date_from=2000-01-01') + + expect(((await to.json()) as { specimens: Array<{ id: number }> }).specimens.map((s) => s.id)).toEqual([specimens[0]!.id]) + expect(((await from.json()) as { specimens: Array<{ id: number }> }).specimens.map((s) => s.id)).toEqual([specimens[1]!.id]) + }) + + it('the specimen list sorts a leftover value after every real date and before a missing one, in either direction', async () => { + const { type, specimens } = await seedStudy(['Apr 2025', undefined, '2024-05-07', '13/2/2024', '1999-12-31']) + const [april, none, may, legacy, early] = specimens.map((s) => s!.id) + + const ids = async (order: string) => { + const res = await ctx.request(`/api/specimens?specimen_type_id=${type.id}&sort=collection_date&order=${order}`) + return ((await res.json()) as { specimens: Array<{ id: number }> }).specimens.map((s) => s.id) + } + + expect(await ids('asc')).toEqual([early, may, legacy, april, none]) + expect(await ids('desc')).toEqual([may, early, april, legacy, none]) + }) + + it('statistics and Container selection match no date bound on a leftover value', async () => { + await seedStudy(['1999-12-31', '13/2/2024', 'Apr 2025']) + + const to = await ctx.request('/api/statistics?collection_date_to=2000-01-01') + const from = await ctx.request('/api/statistics?collection_date_from=2000-01-01') + + expect(((await to.json()) as { specimens: { total: number } }).specimens.total).toBe(1) + expect(((await from.json()) as { specimens: { total: number } }).specimens.total).toBe(0) + }) + + it('a study summary, the study summaries and the study timeline take their date range from real dates only', async () => { + const { studyRecord } = await seedStudy(['2023-01-10', '13/2/2024', '2022-09-07', 'Apr 2025']) + const range = { earliest: '2022-09-07', latest: '2023-01-10' } + + const summary = (await (await ctx.request(`/api/studies/${studyRecord.id}/summary`)).json()) as { + summary: { collectionDateRange: unknown; studyDurationDays: number; collectionTimeline: Array<{ date: string }> } + } + const summaries = (await (await ctx.request(`/api/studies/summaries?ids=${studyRecord.id}`)).json()) as { + summaries: Array<{ collectionDateRange: unknown }> + } + const timeline = (await (await ctx.request(`/api/studies/${studyRecord.id}/timeline`)).json()) as { dateRange: unknown } + + expect(summary.summary.collectionDateRange).toEqual(range) + expect(summary.summary.studyDurationDays).toBe(125) + expect(summary.summary.collectionTimeline.map((t) => t.date)).toEqual(['2022-09-07', '2023-01-10']) + expect(summaries.summaries[0]!.collectionDateRange).toEqual(range) + expect(timeline.dateRange).toEqual(range) + }) + + it('the monthly collection timeline counts real dates only', async () => { + const { specimens } = await seedStudy(['2024-02-01', '5/7/24', '13/2/2024']) + + const result = await computeSpecimenAggregates(ctx.db, specimens, {}) + + expect(result.collectionTimeline).toEqual([{ date: '2024-02', count: 1 }]) + }) + + it('a subject or control batch summary takes its date range from real dates, and places a leftover by its created time', () => { + const specimen = (id: number, collectionDate: string | null, created: string): EnrichedSpecimenSummary => ({ + id, + specimenTypeId: 1, + specimenTypeName: 'Blood', + collectionDate, + created, + lastUpdated: created, + containerCount: 0, + containerBreakdown: {}, + unitBreakdown: {}, + containers: [], + }) + + const summary = buildSpecimenCollectionSummary( + [ + specimen(1, '2024-01-15', '2024-03-01 00:00:00'), + specimen(2, '13/2/2024', '2024-01-01 00:00:00'), + specimen(3, '1/1/20', '2024-02-01 00:00:00'), + ], + [], + new Map(), + ) + + expect(summary.collectionDateRange).toEqual({ earliest: '2024-01-15', latest: '2024-01-15' }) + expect(summary.timeline.map((entry) => entry.id)).toEqual([2, 1, 3]) + }) +}) diff --git a/packages/api/src/routes/studies.ts b/packages/api/src/routes/studies.ts index c22a685b..9eb06ea1 100644 --- a/packages/api/src/routes/studies.ts +++ b/packages/api/src/routes/studies.ts @@ -25,7 +25,7 @@ import { requireParam } from '../lib/common-validators' import { withWriteTransaction } from '../db/write-transaction' import { assertNotUsedInQpcr } from '../lib/qpcr-usage' import { insertedRow } from '../lib/inserted-row' -import { timestampFilter } from '../lib/date-filters' +import { isIsoCalendarDate, timestampFilter } from '../lib/date-filters' import { likeContains } from '../lib/sql-like' /** Short code prefix for tutorial namespace. Any study whose short code starts with this (case-insensitive) may be deleted by any member. Only admins may rename a study into or out of this namespace. */ @@ -396,9 +396,10 @@ studies.get('/:id/summary', authMiddleware, async (c) => { })) // Collection date range - const collectionDates = (specimens + // Only real YYYY-MM-DD dates: migration 008 leaves one it cannot read (`13/2/2024`) as it was. + const collectionDates = specimens .map(s => s.collectionDate) - .filter(Boolean) as string[]) + .filter((date): date is string => date != null && isIsoCalendarDate(date)) .sort() const collectionDateRange = collectionDates.length > 0 @@ -521,7 +522,7 @@ studies.get('/:id/timeline', authMiddleware, async (c) => { id: number studySubjectId: number specimenTypeId: number - collectionDate: string + collectionDate: string | null }> if (specimens.length === 0) { @@ -545,7 +546,7 @@ studies.get('/:id/timeline', authMiddleware, async (c) => { // Group specimens by subject const specimensBySubject = new Map>() @@ -570,7 +571,10 @@ studies.get('/:id/timeline', authMiddleware, async (c) => { })) // Calculate date range - const collectionDates = specimens.map(s => s.collectionDate).sort() + const collectionDates = specimens + .map(s => s.collectionDate) + .filter((date): date is string => date != null && isIsoCalendarDate(date)) + .sort() const dateRange = collectionDates.length > 0 ? { earliest: collectionDates[0], From a06a04d8fc8ee6ffdeecce73f124fe4f2ab1c2ef Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:33:09 -0800 Subject: [PATCH 50/54] fix(db): remove migration 013; the lab's cleanup is not a migration (#126) Migrations ship to every install, and 013 named our production rows and carried out our lab's #124 decisions, so on another install it would either stop startup or change data on our lab's say-so. This reverts d88b559: CURRENT_SCHEMA_VERSION goes back to 12, and initial_schema.sql, schema.ts and the legacy fixtures go back with it. There is no case-insensitive unique index on Subject names: it would stop startup on any install that already holds such names. The write paths from 30ea366 enforce the rule, and the integrity report lists existing pairs. CONTEXT.md, the code comments, the integrity report page and the user guide no longer mention migration 013, our lab's choices or our snapshot's counts. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 6 +- packages/api/initial_schema.sql | 3 +- packages/api/src/db/__tests__/client.test.ts | 11 +- .../src/db/__tests__/migration-013.test.ts | 313 ------------------ .../src/db/__tests__/schema-evolution.test.ts | 15 +- .../db/migrations/013_legacy_data_cleanup.sql | 169 ---------- packages/api/src/db/schema-evolution.ts | 2 +- packages/api/src/db/schema.ts | 2 - packages/api/src/lib/data-audit.ts | 8 +- packages/api/src/lib/validation.ts | 7 +- .../src/routes/__tests__/data-audit.test.ts | 1 - .../docs/guides/advanced/data-integrity.md | 6 +- .../src/pages/AdminDataIntegrityReport.tsx | 2 +- 13 files changed, 25 insertions(+), 520 deletions(-) delete mode 100644 packages/api/src/db/__tests__/migration-013.test.ts delete mode 100644 packages/api/src/db/migrations/013_legacy_data_cleanup.sql diff --git a/CONTEXT.md b/CONTEXT.md index 26cfb14b..564d772f 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -57,15 +57,15 @@ The top-level organizational unit for a research project, clinical trial, or oth _Avoid_: Project (unless referring to non-study work), trial (as a synonym for study) **Subject**: -A participant or sample source within a study, identified by a unique name within that study. Names compare ignoring case (ASCII letters, as SQLite's `NOCASE` does): a study can't have both `ABC-1` and `abc-1`, and creating, renaming or importing a Subject whose name differs from an existing one only by case is refused. The unique index `study_subject_study_name_nocase_idx` enforces it. Decided in issue #124 (decision 6), where the two legacy CTRL pairs were merged by migration 013. +A participant or sample source within a study, identified by a unique name within that study. Names compare ignoring case (ASCII letters, as SQLite's `NOCASE` does): a study can't have both `ABC-1` and `abc-1`, and creating, renaming or importing a Subject whose name differs from an existing one only by case is refused. The write paths enforce it; there is no database index for it, because a database from before the rule may already hold such names, and an index would stop it from starting. The integrity report lists those pairs for an operator to merge or rename. Decided in issue #124 (decision 6). _Avoid_: Patient (unless the study is clinical and that is your org's term), donor (unless that is your org's term) **Specimen**: The collected unit — a registered biological material from a specific collection event. One collection event, one specimen record. **Specimen identity**: -A Subject's Specimen is identified by its Subject, Specimen type and collection date: **Specimen registration** adds Containers to the Specimen with that combination instead of creating a second one. Existing duplicates stay as they are (issue #124, decision 7): the lab chose no merges and no new unique index on production. On the 2026-10-31 production snapshot that leaves 15 dated pairs, which migration 008 created by converting legacy `M/D/YY` dates, and undated groups: 98 before migration 013, and 502 after it cleared the placeholder collection dates (decision 1), because a Specimen dated `1970-01-01` now shares an empty date with its undated twin. The integrity report lists them all. Known difference: fresh installs have the unique index `idx_specimen_study_subject_type_date` on (`study_subject_id`, `specimen_type_id`, `collection_date`) and production does not. That index treats an empty collection date as distinct, so fresh installs also allow undated duplicates. -_Avoid_: Assuming the database refuses a duplicate Specimen; on production only registration's lookup prevents one. +A Subject's Specimen is identified by its Subject, Specimen type and collection date: **Specimen registration** adds Containers to the Specimen with that combination instead of creating a second one. Existing duplicates stay as they are (issue #124, decision 7): no migration merges them or adds a unique index to an upgraded database, and the integrity report lists them. Migration 008 can create dated duplicates by converting legacy `M/D/YY` dates, and clearing a placeholder collection date can make a Specimen an undated duplicate of another. Known difference: fresh installs have the unique index `idx_specimen_study_subject_type_date` on (`study_subject_id`, `specimen_type_id`, `collection_date`) and upgraded databases do not. That index treats an empty collection date as distinct, so fresh installs also allow undated duplicates. +_Avoid_: Assuming the database refuses a duplicate Specimen; on an upgraded database only registration's lookup prevents one. _Avoid_: Sample (as the domain term; see **Sample** below) **Sample**: diff --git a/packages/api/initial_schema.sql b/packages/api/initial_schema.sql index 94aeafb7..ea80d50a 100644 --- a/packages/api/initial_schema.sql +++ b/packages/api/initial_schema.sql @@ -55,7 +55,6 @@ CREATE TABLE IF NOT EXISTS study_subject ( CREATE INDEX IF NOT EXISTS study_subject_study_id_idx ON study_subject(study_id);--> statement-breakpoint CREATE INDEX IF NOT EXISTS study_subject_name_idx ON study_subject(name);--> statement-breakpoint CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_idx ON study_subject(study_id, name);--> statement-breakpoint -CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_nocase_idx ON study_subject(study_id, name COLLATE NOCASE);--> statement-breakpoint CREATE TABLE IF NOT EXISTS control_definition ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE, @@ -298,7 +297,7 @@ CREATE TABLE IF NOT EXISTS storage_type ( CREATE TABLE IF NOT EXISTS schema_version ( version INTEGER NOT NULL );--> statement-breakpoint -INSERT INTO schema_version (version) VALUES (13);--> statement-breakpoint +INSERT INTO schema_version (version) VALUES (12);--> statement-breakpoint CREATE TABLE IF NOT EXISTS settings ( key TEXT NOT NULL, user_id INTEGER REFERENCES users(id), diff --git a/packages/api/src/db/__tests__/client.test.ts b/packages/api/src/db/__tests__/client.test.ts index 6b26b7c5..3ae770fa 100644 --- a/packages/api/src/db/__tests__/client.test.ts +++ b/packages/api/src/db/__tests__/client.test.ts @@ -32,18 +32,15 @@ describe('Database Client', () => { db.exec(`CREATE TABLE micronix_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE cryovial_tube (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, barcode TEXT, position TEXT)`) db.exec(`CREATE TABLE static_well (id INTEGER PRIMARY KEY, collection_id INTEGER NOT NULL, position TEXT)`) - db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL, last_updated TEXT)`) + db.exec(`CREATE TABLE study_subject (id INTEGER PRIMARY KEY, study_id INTEGER NOT NULL, name TEXT NOT NULL)`) db.exec(`CREATE TABLE sheet (id INTEGER PRIMARY KEY, name TEXT NOT NULL, box_id INTEGER, bag_id INTEGER)`) - db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, study_subject_id INTEGER, control_batch_id INTEGER, specimen_type_id INTEGER NOT NULL, collection_date DATE, last_updated TEXT)`) + db.exec(`CREATE TABLE specimen (id INTEGER PRIMARY KEY, study_subject_id INTEGER, control_batch_id INTEGER, specimen_type_id INTEGER NOT NULL, collection_date DATE)`) // Tables migration 011 indexes - db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER, comment TEXT, remaining_quantity REAL, last_updated TEXT)`) + db.exec(`CREATE TABLE storage_container (id INTEGER PRIMARY KEY, specimen_id INTEGER)`) db.exec(`CREATE TABLE storage_container_tag (storage_container_id INTEGER, tag_id INTEGER)`) db.exec(`CREATE TABLE control_batch (id INTEGER PRIMARY KEY, control_definition_id INTEGER, properties TEXT)`) db.exec(`CREATE TABLE location (id INTEGER PRIMARY KEY, parent_id INTEGER, path TEXT)`) - db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER, name TEXT)`) - // Tables migration 013 reads or changes - db.exec(`CREATE TABLE container_derivation (id INTEGER PRIMARY KEY, parent_container_id INTEGER, child_container_id INTEGER)`) - db.exec(`CREATE TABLE qpcr_experiment_well (id INTEGER PRIMARY KEY, storage_container_id INTEGER, specimen_id INTEGER)`) + db.exec(`CREATE TABLE micronix_plate (id INTEGER PRIMARY KEY, location_id INTEGER)`) db.exec(`CREATE TABLE cryovial_box (id INTEGER PRIMARY KEY, location_id INTEGER)`) db.exec(`CREATE TABLE bag (id INTEGER PRIMARY KEY, location_id INTEGER)`) // Intentionally omit paper — migration 003 preflight creates legacy stub diff --git a/packages/api/src/db/__tests__/migration-013.test.ts b/packages/api/src/db/__tests__/migration-013.test.ts deleted file mode 100644 index 5b732e0d..00000000 --- a/packages/api/src/db/__tests__/migration-013.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest' -import { Database } from 'bun:sqlite' -import { applyInitialSchema } from '../apply-initial-schema' -import { CURRENT_SCHEMA_VERSION, evolveOperationalSchema, getRecordedSchemaVersion } from '../schema-evolution' - -/** - * Migration 013 applies the lab's legacy data cleanup decisions (issues #124 and #126). Each test - * builds a database in production's version-12 shape: the current snapshot, without the indexes - * production lacks, seeded with the production ids each decision names. - */ - -const NOW = '2024-01-01 00:00:00' - -function databaseAtVersion12(): Database { - const sqlite = new Database(':memory:') - sqlite.exec('PRAGMA foreign_keys=ON') - applyInitialSchema(sqlite) - sqlite.exec('UPDATE schema_version SET version = 12') - // Production has neither index: no Specimen identity index (decision 7), and migration 013 - // adds the Subject name one. - sqlite.exec('DROP INDEX IF EXISTS idx_specimen_study_subject_type_date') - sqlite.exec('DROP INDEX IF EXISTS study_subject_study_name_nocase_idx') - sqlite.exec(`INSERT INTO study (id, title, short_code, is_longitudinal, lead_person, created, last_updated) VALUES - (3, 'Controls', 'CTRL', 0, 'Lab', '${NOW}', '${NOW}'), (8, 'Jiangsu', 'JIANGSU17', 0, 'Lab', '${NOW}', '${NOW}')`) - sqlite.exec(`INSERT INTO specimen_type (id, name, created, last_updated) VALUES (2, 'DBS', '${NOW}', '${NOW}'), (21, 'WB', '${NOW}', '${NOW}')`) - sqlite.exec(`INSERT INTO unit (id, symbol, name, category) VALUES (5, 'uL', 'microliter', 'volume')`) - sqlite.exec(`INSERT INTO storage_type (id, name) VALUES (1, 'Freezer')`) - sqlite.exec(`INSERT INTO location (id, name, storage_type_id) VALUES (107, 'Freezer A', 1)`) - sqlite.exec(`INSERT INTO cryovial_box (id, location_id, name, created, last_updated) VALUES (900, 107, 'Box 900', '${NOW}', '${NOW}')`) - return sqlite -} - -function addSubject(sqlite: Database, id: number, studyId: number, name: string): void { - sqlite.prepare(`INSERT INTO study_subject (id, study_id, name, created, last_updated) VALUES (?, ?, ?, ?, ?)`).run(id, studyId, name, NOW, NOW) -} - -function addSpecimen(sqlite: Database, id: number, subjectId: number, typeId: number, date: string | null): void { - sqlite - .prepare(`INSERT INTO specimen (id, study_subject_id, specimen_type_id, collection_date, created, last_updated) VALUES (?, ?, ?, ?, ?, ?)`) - .run(id, subjectId, typeId, date, NOW, NOW) -} - -function addContainer(sqlite: Database, id: number, specimenId: number, options: { comment?: string; remaining?: number } = {}): void { - sqlite - .prepare(`INSERT INTO storage_container (id, specimen_id, comment, total_quantity, remaining_quantity, unit_id, created, last_updated) VALUES (?, ?, ?, 1.0, ?, 5, ?, ?)`) - .run(id, specimenId, options.comment ?? null, options.remaining ?? 1.0, NOW, NOW) -} - -function addCryovialTube(sqlite: Database, id: number, barcode: string | null): void { - sqlite.prepare(`INSERT INTO cryovial_tube (id, collection_id, barcode) VALUES (?, 900, ?)`).run(id, barcode) -} - -function addMicronixTube(sqlite: Database, id: number, plateId: number, barcode: string): void { - sqlite.prepare(`INSERT INTO micronix_tube (id, collection_id, barcode) VALUES (?, ?, ?)`).run(id, plateId, barcode) -} - -/** Production's rows for every decision, plus neighbours each step must leave alone. */ -function seedProductionRows(sqlite: Database): void { - addSubject(sqlite, 31047, 3, '3d7_0.05') - addSubject(sqlite, 66537, 3, '3D7_0.05') - addSubject(sqlite, 31049, 3, '3d7_10k') - addSubject(sqlite, 66543, 3, '3D7_10K') - addSubject(sqlite, 25613, 8, 'JS087') - addSubject(sqlite, 500, 8, 'S500') - - // Decision 6: Specimen 43644 has no match on the target and moves to it; Specimen 43645 - // matches target Specimen 120524 (same type and date) and is merged into it. - addSpecimen(sqlite, 43644, 31047, 21, '2018-03-15') - addSpecimen(sqlite, 43645, 31047, 2, '2021-06-07') - addSpecimen(sqlite, 120523, 66537, 2, '2021-06-08') - addSpecimen(sqlite, 120524, 66537, 2, '2021-06-07') - addSpecimen(sqlite, 43646, 31049, 21, '2018-03-15') - addSpecimen(sqlite, 120529, 66543, 2, '2021-06-07') - addContainer(sqlite, 70001, 43644) - addContainer(sqlite, 70002, 43645) - addContainer(sqlite, 70003, 43645) - addContainer(sqlite, 70004, 120524) - addContainer(sqlite, 70005, 43646) - - // Decision 1: placeholder dates, and dates that stay (including a future one, decision 2). - addSpecimen(sqlite, 1, 500, 2, '1970-01-01') - addSpecimen(sqlite, 2, 500, 2, '1999-09-09') - addSpecimen(sqlite, 3, 500, 2, '1899-12-30') - addSpecimen(sqlite, 4, 500, 2, '1111-01-01') - addSpecimen(sqlite, 5, 500, 2, '1970-01-02') - addSpecimen(sqlite, 6, 500, 2, '2073-02-08') - addSpecimen(sqlite, 7, 500, 21, '1970-01-01') - - // Decision 3: exhausted Containers with remaining 1, and ones that stay. - addSpecimen(sqlite, 10, 500, 21, '2020-01-01') - addContainer(sqlite, 50465, 10, { comment: 'exhausted' }) - addContainer(sqlite, 124552, 10, { comment: 'Z32592; exhausted' }) - addContainer(sqlite, 143335, 10, { comment: 'Z20759; Zymo cleaned; exhausted' }) - addContainer(sqlite, 74034, 10, { comment: 'Reuploaded & Set to Exhausted', remaining: 0 }) - addContainer(sqlite, 74035, 10, { comment: 'half used', remaining: 0.5 }) - - // Decision 4: placeholder tube barcodes, and real barcodes that contain NA. - for (const [id, barcode] of [ - [80001, 'NA'], - [80002, 'N/A'], - [80003, 'n/a'], - [80004, ' NA '], - [80005, 'NA98'], - [80006, 'DFNA'], - [80007, null], - ] as const) { - addContainer(sqlite, id, 10) - addCryovialTube(sqlite, id, barcode) - } - - // Decision 5: Container 34513 has no tube or Paper row; its Specimen keeps its other Container. - addSpecimen(sqlite, 31863, 25613, 21, '2018-07-24') - addContainer(sqlite, 34513, 31863) - addContainer(sqlite, 34514, 31863) - addCryovialTube(sqlite, 34514, 'JS087-1') - - // Decision 8: plate 1760 is empty; plate 1761 keeps its tubes. - sqlite.exec(`INSERT INTO micronix_plate (id, location_id, name, created, last_updated) VALUES - (1760, 107, 'EXVIVO-27 ', '${NOW}', '${NOW}'), (1761, 107, 'EXVIVO-27', '${NOW}', '${NOW}')`) - addContainer(sqlite, 90001, 10) - addMicronixTube(sqlite, 90001, 1761, '1000000001') -} - -function column(sqlite: Database, sql: string): T[] { - return (sqlite.prepare(sql).all() as Array>).map((row) => Object.values(row)[0] as T) -} - -function rows(sqlite: Database, sql: string): unknown[] { - return sqlite.prepare(sql).all() -} - -describe('migration 013: legacy data cleanup', () => { - let sqlite: Database - - beforeEach(() => { - sqlite = databaseAtVersion12() - }) - - afterEach(() => { - sqlite.close() - }) - - it('applies each decision to the rows it names and leaves every other row alone', () => { - seedProductionRows(sqlite) - - evolveOperationalSchema(sqlite) - - expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) - // Decision 1 - expect(rows(sqlite, 'SELECT id, collection_date FROM specimen WHERE id <= 7 ORDER BY id')).toEqual([ - { id: 1, collection_date: null }, - { id: 2, collection_date: null }, - { id: 3, collection_date: null }, - { id: 4, collection_date: null }, - { id: 5, collection_date: '1970-01-02' }, - { id: 6, collection_date: '2073-02-08' }, - { id: 7, collection_date: null }, - ]) - // Decision 3 - expect(rows(sqlite, 'SELECT id, comment, remaining_quantity FROM storage_container WHERE specimen_id = 10 AND comment IS NOT NULL ORDER BY id')).toEqual([ - { id: 50465, comment: 'exhausted', remaining_quantity: 0 }, - { id: 74034, comment: 'Reuploaded & Set to Exhausted', remaining_quantity: 0 }, - { id: 74035, comment: 'half used', remaining_quantity: 0.5 }, - { id: 124552, comment: 'Z32592; exhausted', remaining_quantity: 0 }, - { id: 143335, comment: 'Z20759; Zymo cleaned; exhausted', remaining_quantity: 0 }, - ]) - // Decision 4 - expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube WHERE id BETWEEN 80001 AND 80007 ORDER BY id')).toEqual([ - { id: 80001, barcode: null }, - { id: 80002, barcode: null }, - { id: 80003, barcode: null }, - { id: 80004, barcode: null }, - { id: 80005, barcode: 'NA98' }, - { id: 80006, barcode: 'DFNA' }, - { id: 80007, barcode: null }, - ]) - // Decision 5 - expect(column(sqlite, 'SELECT id FROM storage_container WHERE specimen_id = 31863')).toEqual([34514]) - // Decision 8 - expect(column(sqlite, `SELECT id FROM micronix_plate WHERE trim(name) = 'EXVIVO-27'`)).toEqual([1761]) - expect(column(sqlite, 'SELECT COUNT(*) FROM micronix_tube WHERE collection_id = 1761')).toEqual([1]) - // Decision 6: the app's Subject merge, run for both pairs. - expect(rows(sqlite, 'SELECT id, name FROM study_subject WHERE study_id = 3 ORDER BY id')).toEqual([ - { id: 66537, name: '3D7_0.05' }, - { id: 66543, name: '3D7_10K' }, - ]) - expect(rows(sqlite, 'SELECT id, study_subject_id FROM specimen WHERE id IN (43644, 43645, 43646, 120523, 120524, 120529) ORDER BY id')).toEqual([ - { id: 43644, study_subject_id: 66537 }, - { id: 43646, study_subject_id: 66543 }, - { id: 120523, study_subject_id: 66537 }, - { id: 120524, study_subject_id: 66537 }, - { id: 120529, study_subject_id: 66543 }, - ]) - expect(rows(sqlite, 'SELECT id, specimen_id FROM storage_container WHERE id BETWEEN 70001 AND 70005 ORDER BY id')).toEqual([ - { id: 70001, specimen_id: 43644 }, - { id: 70002, specimen_id: 120524 }, - { id: 70003, specimen_id: 120524 }, - { id: 70004, specimen_id: 120524 }, - { id: 70005, specimen_id: 43646 }, - ]) - expect(column(sqlite, 'SELECT last_updated FROM study_subject WHERE id IN (66537, 66543)').every((t) => t !== NOW)).toBe(true) - expect(column(sqlite, 'SELECT last_updated FROM specimen WHERE id = 43644')).not.toEqual([NOW]) - expect(column(sqlite, 'SELECT last_updated FROM storage_container WHERE id = 70002')).not.toEqual([NOW]) - expect(column(sqlite, `SELECT name FROM sqlite_temp_master WHERE name LIKE '__migration_013%'`)).toEqual([]) - expect(column(sqlite, 'PRAGMA foreign_key_check')).toEqual([]) - }) - - it('adds a case-insensitive unique index on Subject name within a Study', () => { - seedProductionRows(sqlite) - - evolveOperationalSchema(sqlite) - - expect(() => addSubject(sqlite, 1, 3, '3d7_10k')).toThrow(/UNIQUE constraint failed/) - expect(() => addSubject(sqlite, 2, 8, 'js087')).toThrow(/UNIQUE constraint failed/) - addSubject(sqlite, 3, 8, '3d7_10k') - expect(column(sqlite, `SELECT COUNT(*) FROM study_subject WHERE name = '3d7_10k'`)).toEqual([1]) - }) - - it('changes nothing on a database without the legacy rows', () => { - addSubject(sqlite, 1, 3, 'A') - addSpecimen(sqlite, 1, 1, 2, '2020-01-01') - addContainer(sqlite, 1, 1, { comment: 'exhausted', remaining: 0 }) - addCryovialTube(sqlite, 1, 'CV1') - - evolveOperationalSchema(sqlite) - - expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) - expect(rows(sqlite, 'SELECT id, study_subject_id, collection_date FROM specimen')).toEqual([{ id: 1, study_subject_id: 1, collection_date: '2020-01-01' }]) - expect(rows(sqlite, 'SELECT id, remaining_quantity FROM storage_container')).toEqual([{ id: 1, remaining_quantity: 0 }]) - expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube')).toEqual([{ id: 1, barcode: 'CV1' }]) - expect(column(sqlite, 'SELECT COUNT(*) FROM study_subject')).toEqual([1]) - }) - - const aborts: Array<[string, (sqlite: Database) => void, RegExp]> = [ - [ - 'an exhausted Container the lab did not review still has quantity', - (db) => addContainer(db, 99999, 10, { comment: 'exhausted' }), - /exhausted/, - ], - [ - 'a reviewed exhausted Container has a remaining quantity other than 1', - (db) => db.exec('UPDATE storage_container SET remaining_quantity = 0.5 WHERE id = 50465'), - /exhausted/, - ], - [ - 'there are more Specimens on a placeholder date than the lab reviewed', - (db) => addSpecimen(db, 99999, 500, 21, '1111-01-01'), - /placeholder collection date/, - ], - [ - 'a micronix tube has a placeholder barcode, which cannot be emptied', - (db) => { - addContainer(db, 99999, 10) - addMicronixTube(db, 99999, 1761, 'N/A') - }, - /placeholder barcode/, - ], - [ - 'Container 34513 has a tube row', - (db) => addCryovialTube(db, 34513, 'X34513'), - /Container 34513/, - ], - [ - 'Container 34513 is the only Container of its Specimen', - (db) => db.exec('DELETE FROM cryovial_tube WHERE id = 34514; DELETE FROM storage_container WHERE id = 34514'), - /Container 34513/, - ], - [ - 'plate 1760 holds a tube', - (db) => { - addContainer(db, 99999, 10) - addMicronixTube(db, 99999, 1760, '1000000002') - }, - /plate 1760/, - ], - [ - 'plate 1760 has been renamed', - (db) => db.exec(`UPDATE micronix_plate SET name = 'EXVIVO-28' WHERE id = 1760`), - /plate 1760/, - ], - [ - 'a Subject to merge has been renamed', - (db) => db.exec(`UPDATE study_subject SET name = '3d7_10k_b' WHERE id = 31049`), - /merge/, - ], - [ - 'a Subject to merge has lost its target', - (db) => db.exec(`UPDATE specimen SET study_subject_id = 31049 WHERE study_subject_id = 66543; DELETE FROM study_subject WHERE id = 66543`), - /merge/, - ], - [ - 'another Study has Subjects whose names differ only by case', - (db) => { - addSubject(db, 600, 8, 'abc') - addSubject(db, 601, 8, 'ABC') - }, - /differ only by case/, - ], - ] - - for (const [situation, change, message] of aborts) { - it(`aborts, changing nothing, when ${situation}`, () => { - seedProductionRows(sqlite) - change(sqlite) - const before = sqlite.serialize() - - expect(() => evolveOperationalSchema(sqlite)).toThrow(message) - - expect(getRecordedSchemaVersion(sqlite)).toBe(12) - expect(Buffer.from(sqlite.serialize()).equals(Buffer.from(before))).toBe(true) - }) - } -}) diff --git a/packages/api/src/db/__tests__/schema-evolution.test.ts b/packages/api/src/db/__tests__/schema-evolution.test.ts index c7cc7796..f3d3da92 100644 --- a/packages/api/src/db/__tests__/schema-evolution.test.ts +++ b/packages/api/src/db/__tests__/schema-evolution.test.ts @@ -65,13 +65,6 @@ function createTablesForLaterMigrations(sqlite: Database): void { ensureColumns(sqlite, 'micronix_plate', ['location_id INTEGER']) ensureColumns(sqlite, 'cryovial_box', ['location_id INTEGER']) ensureColumns(sqlite, 'bag', ['location_id INTEGER']) - // Migration 013 reads or changes these columns. - ensureColumns(sqlite, 'study_subject', ['last_updated TEXT']) - ensureColumns(sqlite, 'specimen', ['last_updated TEXT']) - ensureColumns(sqlite, 'storage_container', ['comment TEXT', 'remaining_quantity REAL', 'last_updated TEXT']) - ensureColumns(sqlite, 'micronix_plate', ['name TEXT']) - ensureColumns(sqlite, 'container_derivation', ['parent_container_id INTEGER', 'child_container_id INTEGER']) - ensureColumns(sqlite, 'qpcr_experiment_well', ['storage_container_id INTEGER', 'specimen_id INTEGER']) // Before version 3, migration 003 renames barcode to sublabel (and its preflight creates the // table if missing), so only a database already past 003 gets paper in its later shape. const version = hasSchemaVersionTable(sqlite) ? getRecordedSchemaVersion(sqlite) : null @@ -289,13 +282,13 @@ describe('schema evolution', () => { sqlite.exec('CREATE TABLE schema_version (version INTEGER NOT NULL)') sqlite.exec('INSERT INTO schema_version (version) VALUES (4)') createTablesForLaterMigrations(sqlite) - sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (1, 1, ' S1 '), (2, 1, 'S2'), (3, 2, 'S1')`) + sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (1, 1, ' S1 '), (2, 1, 's1'), (3, 2, 'S1')`) evolveOperationalSchema(sqlite) expect(getRecordedSchemaVersion(sqlite)).toBe(CURRENT_SCHEMA_VERSION) const names = sqlite.prepare('SELECT name FROM study_subject ORDER BY id').all() as Array<{ name: string }> - expect(names.map((r) => r.name)).toEqual(['S1', 'S2', 'S1']) + expect(names.map((r) => r.name)).toEqual(['S1', 's1', 'S1']) expect(() => sqlite.exec(`INSERT INTO study_subject (id, study_id, name) VALUES (4, 1, 'S1')`)).toThrow(/UNIQUE/) sqlite.close() }) @@ -355,7 +348,7 @@ describe('schema evolution', () => { [6, null], [7, ' '], [8, ' 3/17/22 '], - [9, '1970-01-02'], + [9, '1970-01-01'], ] const insert = sqlite.prepare('INSERT INTO specimen (id, specimen_type_id, collection_date) VALUES (?, 1, ?)') for (const [id, date] of values) insert.run(id, date) @@ -373,7 +366,7 @@ describe('schema evolution', () => { null, null, '2022-03-17', - '1970-01-02', + '1970-01-01', ]) sqlite.close() }) diff --git a/packages/api/src/db/migrations/013_legacy_data_cleanup.sql b/packages/api/src/db/migrations/013_legacy_data_cleanup.sql deleted file mode 100644 index 9113b8d0..00000000 --- a/packages/api/src/db/migrations/013_legacy_data_cleanup.sql +++ /dev/null @@ -1,169 +0,0 @@ --- Migration 013: apply the lab's legacy data cleanup decisions (issue #124, "Decisions (2026-11-01)"; --- issue #126). Each step changes only the rows its decision names, found by the decision's own rule --- and checked against what the production snapshot of 2026-10-31 held. If a database holds rows the --- lab did not review, the migration aborts and changes nothing, so they can be reviewed first. --- 1. Placeholder collection dates (1970-01-01, 1999-09-09, 1899-12-30, 1111-01-01) become unknown. --- 3. Containers commented "exhausted" with remaining quantity 1 get remaining quantity 0; the --- comment stays. --- 4. NA / N/A-style cryovial tube barcodes become empty (NULL, as for any tube without a barcode). --- 5. Container 34513 (no tube or Paper row; Specimen 31863 keeps its other Container) is deleted. --- 6. CTRL Subjects 3d7_0.05 (31047) and 3d7_10k (31049) are merged into 3D7_0.05 (66537) and --- 3D7_10K (66543) the way the app's Subject merge does it, and Subject names become unique --- within a Study ignoring case. --- 8. Empty plate 1760 `EXVIVO-27 ` (trailing space) is deleted; plate 1761 `EXVIVO-27` stays. --- Decisions 2 (future dates) and 7 (duplicate Specimen identity) change no data; the integrity --- report lists them. -CREATE TEMP TABLE __migration_013_exhausted (id INTEGER PRIMARY KEY);--> statement-breakpoint -INSERT INTO __migration_013_exhausted (id) VALUES - (50465), (87730), (87732), (87735), (87736), (87737), (87738), (87739), (87741), (87742), - (87743), (87744), (87747), (87748), (87749), (87750), (87751), (87752), (87753), (87754), - (87756), (87757), (87759), (87762), (87763), (87765), (87766), (87768), (87769), (124552), - (124564), (124565), (124568), (124570), (124598), (124645), (124646), (136248), (140979), (143331), - (143335), (143339), (143343), (143347), (144277), (144318);--> statement-breakpoint -CREATE TEMP TABLE __migration_013_placeholder_dates (collection_date TEXT PRIMARY KEY, reviewed INTEGER NOT NULL);--> statement-breakpoint -INSERT INTO __migration_013_placeholder_dates (collection_date, reviewed) VALUES - ('1970-01-01', 17043), ('1999-09-09', 245), ('1899-12-30', 17), ('1111-01-01', 1);--> statement-breakpoint -CREATE TEMP TABLE __migration_013_subject_merge ( - source_id INTEGER PRIMARY KEY, - source_name TEXT NOT NULL, - target_id INTEGER NOT NULL, - target_name TEXT NOT NULL -);--> statement-breakpoint -INSERT INTO __migration_013_subject_merge (source_id, source_name, target_id, target_name) VALUES - (31047, '3d7_0.05', 66537, '3D7_0.05'), (31049, '3d7_10k', 66543, '3D7_10K');--> statement-breakpoint --- A pair is merged only while its source Subject still exists. -DELETE FROM __migration_013_subject_merge WHERE source_id NOT IN (SELECT id FROM study_subject);--> statement-breakpoint -CREATE TEMP TABLE IF NOT EXISTS __migration_013_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint -CREATE TRIGGER __migration_013_dates_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot clear placeholder collection dates: more Specimens are dated 1970-01-01, 1999-09-09, 1899-12-30 or 1111-01-01 than the lab reviewed (17043, 245, 17 and 1). Review the newer ones first: SELECT id, collection_date, created FROM specimen WHERE collection_date IN (''1970-01-01'', ''1999-09-09'', ''1899-12-30'', ''1111-01-01'') ORDER BY created DESC.') - WHERE EXISTS ( - SELECT 1 FROM specimen s JOIN __migration_013_placeholder_dates p ON p.collection_date = s.collection_date - GROUP BY p.collection_date HAVING COUNT(*) > MAX(p.reviewed) - ); -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_exhausted_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot set exhausted Containers to remaining quantity 0: a Container commented exhausted has remaining quantity above 0 and is not one of the 46 the lab reviewed, or its remaining quantity is not 1. Review it first: SELECT id, comment, remaining_quantity FROM storage_container WHERE comment LIKE ''%exhausted%'' AND remaining_quantity > 0.') - WHERE EXISTS ( - SELECT 1 FROM storage_container - WHERE comment LIKE '%exhausted%' AND remaining_quantity > 0 - AND (id NOT IN (SELECT id FROM __migration_013_exhausted) OR remaining_quantity <> 1) - ); -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_barcodes_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot empty placeholder barcodes: a micronix tube has an NA-style placeholder barcode (micronix tubes must have one), or more cryovial tubes have one than the 285 the lab reviewed. Review them first.') - WHERE EXISTS ( - SELECT 1 FROM micronix_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' - ) OR ( - SELECT COUNT(*) FROM cryovial_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' - ) > 285; -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_container_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot delete Container 34513: it no longer matches what the lab reviewed (Specimen 31863, no tube, Paper or well row, no derivation, Tag or qPCR well, and its Specimen has another Container). Review it first.') - WHERE EXISTS (SELECT 1 FROM storage_container WHERE id = 34513) - AND NOT ( - (SELECT specimen_id FROM storage_container WHERE id = 34513) = 31863 - AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE id = 34513) - AND NOT EXISTS (SELECT 1 FROM cryovial_tube WHERE id = 34513) - AND NOT EXISTS (SELECT 1 FROM paper WHERE id = 34513) - AND NOT EXISTS (SELECT 1 FROM static_well WHERE id = 34513) - AND NOT EXISTS (SELECT 1 FROM container_derivation WHERE parent_container_id = 34513 OR child_container_id = 34513) - AND NOT EXISTS (SELECT 1 FROM storage_container_tag WHERE storage_container_id = 34513) - AND NOT EXISTS (SELECT 1 FROM qpcr_experiment_well WHERE storage_container_id = 34513) - AND EXISTS (SELECT 1 FROM storage_container WHERE specimen_id = 31863 AND id <> 34513) - ); -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_plate_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot delete plate 1760: it no longer matches what the lab reviewed (named "EXVIVO-27 " with a trailing space, holding no tubes or wells, beside plate 1761 "EXVIVO-27"). Review it first.') - WHERE EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1760) - AND NOT ( - (SELECT name FROM micronix_plate WHERE id = 1760) = 'EXVIVO-27 ' - AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE collection_id = 1760) - AND NOT EXISTS (SELECT 1 FROM static_well WHERE collection_id = 1760) - AND EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1761 AND name = 'EXVIVO-27') - ); -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_merge_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot merge the CTRL Subjects 3d7_0.05 into 3D7_0.05 and 3d7_10k into 3D7_10K: one of them no longer has the name or Study the lab reviewed, or its target Subject is missing. Review them first.') - WHERE EXISTS ( - SELECT 1 FROM __migration_013_subject_merge m - JOIN study_subject src ON src.id = m.source_id - LEFT JOIN study_subject tgt ON tgt.id = m.target_id - WHERE src.name IS NOT m.source_name - OR tgt.name IS NOT m.target_name - OR tgt.study_id IS NOT src.study_id - OR src.study_id IS NOT (SELECT id FROM study WHERE short_code = 'CTRL') - ); -END;--> statement-breakpoint -CREATE TRIGGER __migration_013_subject_case_tr BEFORE INSERT ON __migration_013_guard -BEGIN - SELECT RAISE(ABORT, 'Cannot make Subject names unique ignoring case: a Study has Subjects whose names differ only by case, beyond the two CTRL pairs the lab chose to merge. Merge or rename them first. SELECT study_id, group_concat(id || '':'' || name, '', '') FROM study_subject GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 lists them.') - WHERE EXISTS ( - SELECT 1 FROM study_subject - WHERE id NOT IN (SELECT source_id FROM __migration_013_subject_merge) - GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 - ); -END;--> statement-breakpoint -INSERT INTO __migration_013_guard DEFAULT VALUES;--> statement-breakpoint -DROP TRIGGER __migration_013_dates_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_exhausted_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_barcodes_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_container_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_plate_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_merge_tr;--> statement-breakpoint -DROP TRIGGER __migration_013_subject_case_tr;--> statement-breakpoint -DROP TABLE __migration_013_guard;--> statement-breakpoint --- Decision 1 -UPDATE specimen SET collection_date = NULL -WHERE collection_date IN (SELECT collection_date FROM __migration_013_placeholder_dates);--> statement-breakpoint --- Decision 3 -UPDATE storage_container SET remaining_quantity = 0 -WHERE id IN (SELECT id FROM __migration_013_exhausted) AND comment LIKE '%exhausted%' AND remaining_quantity = 1;--> statement-breakpoint --- Decision 4 -UPDATE cryovial_tube SET barcode = NULL -WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA';--> statement-breakpoint --- Decision 5 -DELETE FROM storage_container WHERE id = 34513;--> statement-breakpoint --- Decision 8 -DELETE FROM micronix_plate WHERE id = 1760;--> statement-breakpoint --- Decision 6, as mergeSubjects in src/lib/subjects/merge.ts does it. Each source Specimen in turn --- (lowest id first) is merged into the first target Specimen with the same Specimen type and --- collection date, moving its Containers there; a source Specimen with no such match moves to --- the target Subject, where a later source Specimen can match it. Runs after decision 1, so it --- compares the cleaned dates. -CREATE TEMP TABLE __migration_013_specimen_merge AS -SELECT s.id AS specimen_id, m.target_id, - COALESCE( - (SELECT MIN(t.id) FROM specimen t - WHERE t.study_subject_id = m.target_id AND t.specimen_type_id = s.specimen_type_id AND t.collection_date IS s.collection_date), - (SELECT MIN(e.id) FROM specimen e - WHERE e.study_subject_id = m.source_id AND e.id < s.id AND e.specimen_type_id = s.specimen_type_id AND e.collection_date IS s.collection_date) - ) AS match_id -FROM specimen s -JOIN __migration_013_subject_merge m ON m.source_id = s.study_subject_id;--> statement-breakpoint -UPDATE storage_container -SET specimen_id = (SELECT match_id FROM __migration_013_specimen_merge WHERE specimen_id = storage_container.specimen_id), - last_updated = datetime('now') -WHERE specimen_id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint -UPDATE qpcr_experiment_well -SET specimen_id = (SELECT match_id FROM __migration_013_specimen_merge WHERE specimen_id = qpcr_experiment_well.specimen_id) -WHERE specimen_id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint -DELETE FROM specimen -WHERE id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint -UPDATE specimen -SET study_subject_id = (SELECT target_id FROM __migration_013_specimen_merge WHERE specimen_id = specimen.id), - last_updated = datetime('now') -WHERE id IN (SELECT specimen_id FROM __migration_013_specimen_merge WHERE match_id IS NULL);--> statement-breakpoint -UPDATE study_subject SET last_updated = datetime('now') -WHERE id IN (SELECT target_id FROM __migration_013_subject_merge);--> statement-breakpoint -DELETE FROM study_subject WHERE id IN (SELECT source_id FROM __migration_013_subject_merge);--> statement-breakpoint -CREATE UNIQUE INDEX IF NOT EXISTS study_subject_study_name_nocase_idx ON study_subject(study_id, name COLLATE NOCASE);--> statement-breakpoint -DROP TABLE __migration_013_specimen_merge;--> statement-breakpoint -DROP TABLE __migration_013_subject_merge;--> statement-breakpoint -DROP TABLE __migration_013_placeholder_dates;--> statement-breakpoint -DROP TABLE __migration_013_exhausted; diff --git a/packages/api/src/db/schema-evolution.ts b/packages/api/src/db/schema-evolution.ts index 476c0eea..400fe525 100644 --- a/packages/api/src/db/schema-evolution.ts +++ b/packages/api/src/db/schema-evolution.ts @@ -8,7 +8,7 @@ import { listNumberedMigrations, runSqlMigration } from './migration-runner' const PAPER_SUBLABEL_MIGRATION_VERSION = 3 /** Canonical schema level; bump when adding numbered deltas under migrations/. */ -export const CURRENT_SCHEMA_VERSION = 13 +export const CURRENT_SCHEMA_VERSION = 12 const SCHEMA_VERSION_TABLE = 'schema_version' const LEGACY_BASELINE_VERSION = 1 diff --git a/packages/api/src/db/schema.ts b/packages/api/src/db/schema.ts index 506d7689..0a4c2300 100644 --- a/packages/api/src/db/schema.ts +++ b/packages/api/src/db/schema.ts @@ -110,8 +110,6 @@ export const studySubject = sqliteTable('study_subject', { studyIdIdx: index('study_subject_study_id_idx').on(table.studyId), nameIdx: index('study_subject_name_idx').on(table.name), studyNameUniq: uniqueIndex('study_subject_study_name_idx').on(table.studyId, table.name), - // Subject names are unique within a Study ignoring case (CONTEXT.md, Subject; migration 013). - studyNameNocaseUniq: uniqueIndex('study_subject_study_name_nocase_idx').on(table.studyId, sql`${table.name} COLLATE NOCASE`), })) // Control Production Hierarchy diff --git a/packages/api/src/lib/data-audit.ts b/packages/api/src/lib/data-audit.ts index e4628abc..4984ca5b 100644 --- a/packages/api/src/lib/data-audit.ts +++ b/packages/api/src/lib/data-audit.ts @@ -365,7 +365,7 @@ export interface ContainerWithInvalidGridPositionItem { /** * Subject Specimens that share a Subject, Specimen type and collection date (CONTEXT.md, - * Specimen identity). The lab chose to keep the legacy groups (issue #124, decision 7). + * Specimen identity). Existing groups stay until someone merges them by hand. */ export interface DuplicateSpecimenIdentityItem { studyId: number @@ -820,8 +820,8 @@ export async function listExhaustedContainersWithRemainingQuantity(db: Database) } /** - * Collection dates the legacy application stored for "unknown". Migration 013 cleared them - * (issue #124, decision 1); the report flags any that come back. + * Dates commonly stored for an unknown collection date: the Unix epoch, the spreadsheet zero + * date, and two fill-in values legacy tools used. A real collection is rarely on one of them. */ export const PLACEHOLDER_COLLECTION_DATES = ['1970-01-01', '1999-09-09', '1899-12-30', '1111-01-01'] as const @@ -837,7 +837,7 @@ export async function listSpecimensWithPlaceholderCollectionDate(db: Database): /** * Specimens with a YYYY-MM-DD collection date after today on the server. New dates in the future - * are refused on write; the lab corrects the legacy ones by hand (issue #124, decision 2). + * are refused on write; ones stored earlier are corrected by hand. */ export async function listSpecimensWithFutureCollectionDate(db: Database): Promise { const rows = await db diff --git a/packages/api/src/lib/validation.ts b/packages/api/src/lib/validation.ts index 8f49be9a..26a22fb2 100644 --- a/packages/api/src/lib/validation.ts +++ b/packages/api/src/lib/validation.ts @@ -5,8 +5,9 @@ import { resolveStudyByShortCode, resolveSubjectByNameAndStudy, resolveSpecimenT /** * Validate that a subject name is unique within a study. Names compare ignoring case - * (CONTEXT.md, Subject), as the `study_subject_study_name_nocase_idx` index does, so a name that - * differs from an existing one only by case is refused. Pass `excludeSubjectId` when renaming, + * (CONTEXT.md, Subject), as SQLite's NOCASE collation does, so a name that differs from an + * existing one only by case is refused. No database index enforces this (existing installs may + * hold such names, which the integrity report lists), so every write path must check here. Pass `excludeSubjectId` when renaming, * so a Subject may change the case of its own name. */ export async function validateSubjectName( @@ -25,7 +26,7 @@ export async function validateSubjectName( return { valid: false, error: 'Subject name cannot exceed 255 characters' } } - // COLLATE NOCASE matches the unique index, so this refuses exactly what the index would. + // COLLATE NOCASE folds ASCII letters only, as foldNameCase does for batch checks. const existing = await database .select({ id: studySubject.id, name: studySubject.name }) .from(studySubject) diff --git a/packages/api/src/routes/__tests__/data-audit.test.ts b/packages/api/src/routes/__tests__/data-audit.test.ts index 029b0d33..968dcd73 100644 --- a/packages/api/src/routes/__tests__/data-audit.test.ts +++ b/packages/api/src/routes/__tests__/data-audit.test.ts @@ -588,7 +588,6 @@ describe('Data audit API', () => { }) it('lists Subjects in one Study whose names differ only by case', async () => { - ctx.sqlite.exec('DROP INDEX study_subject_study_name_nocase_idx') const ctrl = await createTestStudy(ctx.db, { title: 'Controls', shortCode: 'CTRL' }) const other = await createTestStudy(ctx.db, { title: 'Other', shortCode: 'OTH' }) const upper = await createTestStudySubject(ctx.db, { studyId: ctrl.id, name: '3D7_10K' }) diff --git a/packages/docs/src/content/docs/guides/advanced/data-integrity.md b/packages/docs/src/content/docs/guides/advanced/data-integrity.md index d7322c3c..24a69e78 100644 --- a/packages/docs/src/content/docs/guides/advanced/data-integrity.md +++ b/packages/docs/src/content/docs/guides/advanced/data-integrity.md @@ -28,12 +28,12 @@ The **Integrity report** sub-page lists rows that failed one or more checks. Eac - **Derivation broken references.** container_derivation rows with missing parent or child container. - **Storage container tag orphans.** Tag links that reference a deleted container or tag. - **Duplicate tube barcodes.** Micronix and cryovial tubes that share a barcode, comparing without case or surrounding spaces, so `CK3-3FR4` and `ck3-3fr4` count as one barcode. **Spellings** lists each stored form when they differ. Moves and derivations refuse to pick between these tubes, so identify them by container id or position. -- **Duplicate Specimens.** Specimens of one Subject with the same Specimen type and collection date, dated or undated. Registration treats these as one collection event and adds new Containers to the existing Specimen, but legacy data has groups the lab chose to keep. Fresh installs refuse a second dated Specimen; upgraded databases do not. -- **Subject names that differ only by case.** Subjects in one study whose names are the same ignoring case, such as `3d7_10k` and `3D7_10K`. Subject names in a study ignore case, so the app refuses new ones. Merge or rename the Subjects. +- **Duplicate Specimens.** Specimens of one Subject with the same Specimen type and collection date, dated or undated. Registration treats these as one collection event and adds new Containers to the existing Specimen, but older data can hold groups. Merge them by hand if they are one collection event. Fresh installs refuse a second dated Specimen; upgraded databases do not. +- **Subject names that differ only by case.** Subjects in one study whose names are the same ignoring case, such as `abc-1` and `ABC-1`. Subject names in a study ignore case, so the app refuses new ones. Merge or rename the Subjects. - **Collection names that differ only by case or spaces.** Plates, boxes, or bags of one type, or sheets in one box or bag, whose names are the same ignoring case and surrounding spaces. Names are quoted to show the spaces. Rename one, or delete it if it is empty. - **Containers in a type their Specimen type does not allow.** Containers whose Container type is not one of the allowed Container types of their Specimen type. Allow the type for the Specimen type in Reference data, or correct the Container. - **Exhausted Containers with remaining quantity.** Containers whose comment says `exhausted` but whose remaining quantity is above 0, so they show as In Use. Set the remaining quantity to 0 if the material is used up. -- **Placeholder collection dates.** Specimens dated `1970-01-01`, `1999-09-09`, `1899-12-30` or `1111-01-01`, dates the old application stored for an unknown date. Clear the date or enter the real one. An upgrade to schema version 13 clears the legacy ones. +- **Placeholder collection dates.** Specimens dated `1970-01-01`, `1999-09-09`, `1899-12-30` or `1111-01-01`, dates often stored when the real date was unknown. Clear the date or enter the real one. - **Collection dates in the future.** Specimens whose collection date is after today. New dates in the future are refused; correct the legacy ones by hand. - **Location path inconsistencies.** Locations whose stored `path` does not match the path computed from the parent chain. - **Collection date not ISO.** Specimens whose collection date is not a real date written `YYYY-MM-DD`. Date filters, exports, and study summaries compare collection dates as text, so these Specimens are missed or placed in the wrong range. Correct the date in the database. diff --git a/packages/web/src/pages/AdminDataIntegrityReport.tsx b/packages/web/src/pages/AdminDataIntegrityReport.tsx index 2466c307..af017be4 100644 --- a/packages/web/src/pages/AdminDataIntegrityReport.tsx +++ b/packages/web/src/pages/AdminDataIntegrityReport.tsx @@ -364,7 +364,7 @@ export default function AdminDataIntegrityReport() {
From 364267f2f0788b2759c89c24eb513caf1be924f5 Mon Sep 17 00:00:00 2001 From: Maxwell Murphy Date: Sun, 1 Nov 2026 08:33:33 -0800 Subject: [PATCH 51/54] feat(db): one-time data fix runner, with the lab's cleanup as a fix file (#126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `bun run data-fix [--db=…] [--dry-run]` in packages/api runs a site-specific SQL file against a database by hand. It refuses a database not at CURRENT_SCHEMA_VERSION, copies the database first with VACUUM INTO (.pre-data-fix--.bak, sharing migration-backup.ts), runs the statements in one BEGIN IMMEDIATE transaction with foreign keys on, and records name, checksum and time in data_fix_log, a table the runner creates in the fix's transaction rather than a numbered migration. A second run does nothing and says when the fix ran; a changed file under the same name is refused. --dry-run rolls back and prints the rows each statement would change. ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql carries the former migration 013's decisions 1, 3, 4, 5, 6 and 8 from #124 with its guards, minus the Subject name index, and a header saying it is only for the EPPIcenter database. Its tests moved to src/db/__tests__/data-fixes/. The db README, the deployment and backup guides, and CONTEXT.md (Data fix) document the runner and say other installs must not run our fixes. Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 4 + .../2026-11-01-eppicenter-legacy-cleanup.sql | 199 ++++++++++++ packages/api/package.json | 3 +- packages/api/src/db/README.md | 22 ++ .../api/src/db/__tests__/data-fix.test.ts | 127 ++++++++ .../eppicenter-legacy-cleanup.test.ts | 305 ++++++++++++++++++ packages/api/src/db/data-fix.ts | 180 +++++++++++ packages/api/src/db/migration-backup.ts | 37 ++- .../src/scripts/__tests__/data-fix.test.ts | 92 ++++++ packages/api/src/scripts/data-fix.ts | 84 +++++ .../guides/advanced/backup-and-restore.md | 2 + .../docs/guides/advanced/deployment.md | 31 ++ 12 files changed, 1076 insertions(+), 10 deletions(-) create mode 100644 ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql create mode 100644 packages/api/src/db/__tests__/data-fix.test.ts create mode 100644 packages/api/src/db/__tests__/data-fixes/eppicenter-legacy-cleanup.test.ts create mode 100644 packages/api/src/db/data-fix.ts create mode 100644 packages/api/src/scripts/__tests__/data-fix.test.ts create mode 100644 packages/api/src/scripts/data-fix.ts diff --git a/CONTEXT.md b/CONTEXT.md index 564d772f..2dd05696 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -258,6 +258,10 @@ _Avoid_: Stuffing paper spot ids into `collection.barcode`; sending JSON `null` The application — a specimen inventory and workflow system for research and clinical labs. _Avoid_: LIMS (unless describing the broader category externally), ELN, SRA submission system (not available today) +**Data fix**: +A one-time SQL change that one install's operator runs by hand against that install's database with `bun run data-fix` in `packages/api`, to carry out decisions about that install's own data. Unlike a numbered migration, it ships to no other install, so it may name that install's rows by id. The runner copies the database first, refuses a database below the current schema version, runs the fix in one transaction, and records it in `data_fix_log` so it runs only once. The EPPIcenter lab's fixes live in `ops/data-fixes/`. +_Avoid_: Migration (for a change to one install's data); a migration changes every install's schema. + ## Scope What SampleDB is and is not — for product descriptions and documentation. diff --git a/ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql b/ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql new file mode 100644 index 00000000..36e6dee4 --- /dev/null +++ b/ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql @@ -0,0 +1,199 @@ +-- One-time data fix for the EPPIcenter lab's SampleDB database ONLY. +-- +-- DO NOT RUN THIS ON ANY OTHER DATABASE. It names rows by the ids they have in the EPPIcenter +-- production database (snapshot arwen_2026-10-31) and carries out decisions that lab made about +-- its own legacy data. On another install it would abort or, worse, change unrelated rows. +-- +-- Decisions: eppicenter/sampledb3 issue #124, comment "Decisions (2026-11-01)"; work in #126. +-- Run with `bun run data-fix` (packages/api/src/db/README.md, "One-time data fixes"). Every +-- step is guarded: if the database holds rows the lab did not review, the fix aborts and +-- changes nothing, so they can be reviewed first. +-- +-- 1. Placeholder collection dates (1970-01-01, 1999-09-09, 1899-12-30, 1111-01-01) become unknown. +-- 3. Containers commented "exhausted" with remaining quantity 1 get remaining quantity 0; the +-- comment stays. +-- 4. NA / N/A-style cryovial tube barcodes become empty (NULL, as for any tube without a barcode). +-- 5. Container 34513 (no tube or Paper row; Specimen 31863 keeps its other Container) is deleted. +-- 6. CTRL Subjects 3d7_0.05 (31047) and 3d7_10k (31049) are merged into 3D7_0.05 (66537) and +-- 3D7_10K (66543) the way the app's Subject merge does it. The app refuses new case-variant +-- Subject names; there is no database index for it. +-- 8. Empty plate 1760 `EXVIVO-27 ` (trailing space) is deleted; plate 1761 `EXVIVO-27` stays. +-- Decisions 2 (future dates) and 7 (duplicate Specimen identity) change no data; the integrity +-- report lists them. + +-- Setup: list the exhausted Containers the lab reviewed (decision 3) +CREATE TEMP TABLE __fix_exhausted (id INTEGER PRIMARY KEY);--> statement-breakpoint +-- Setup: reviewed exhausted Container ids +INSERT INTO __fix_exhausted (id) VALUES + (50465), (87730), (87732), (87735), (87736), (87737), (87738), (87739), (87741), (87742), + (87743), (87744), (87747), (87748), (87749), (87750), (87751), (87752), (87753), (87754), + (87756), (87757), (87759), (87762), (87763), (87765), (87766), (87768), (87769), (124552), + (124564), (124565), (124568), (124570), (124598), (124645), (124646), (136248), (140979), (143331), + (143335), (143339), (143343), (143347), (144277), (144318);--> statement-breakpoint +-- Setup: list the placeholder dates and the counts the lab reviewed (decision 1) +CREATE TEMP TABLE __fix_placeholder_dates (collection_date TEXT PRIMARY KEY, reviewed INTEGER NOT NULL);--> statement-breakpoint +-- Setup: placeholder dates and reviewed counts +INSERT INTO __fix_placeholder_dates (collection_date, reviewed) VALUES + ('1970-01-01', 17043), ('1999-09-09', 245), ('1899-12-30', 17), ('1111-01-01', 1);--> statement-breakpoint +-- Setup: list the Subject pairs to merge (decision 6) +CREATE TEMP TABLE __fix_subject_merge ( + source_id INTEGER PRIMARY KEY, + source_name TEXT NOT NULL, + target_id INTEGER NOT NULL, + target_name TEXT NOT NULL +);--> statement-breakpoint +-- Setup: Subject pairs to merge +INSERT INTO __fix_subject_merge (source_id, source_name, target_id, target_name) VALUES + (31047, '3d7_0.05', 66537, '3D7_0.05'), (31049, '3d7_10k', 66543, '3D7_10K');--> statement-breakpoint +-- Setup: drop pairs whose source Subject is gone +-- A pair is merged only while its source Subject still exists. +DELETE FROM __fix_subject_merge WHERE source_id NOT IN (SELECT id FROM study_subject);--> statement-breakpoint +-- Guard: table the guard triggers fire on +CREATE TEMP TABLE IF NOT EXISTS __fix_guard (id INTEGER PRIMARY KEY);--> statement-breakpoint +-- Guard: placeholder date counts (decision 1) +CREATE TRIGGER __fix_dates_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot clear placeholder collection dates: more Specimens are dated 1970-01-01, 1999-09-09, 1899-12-30 or 1111-01-01 than the lab reviewed (17043, 245, 17 and 1). Review the newer ones first: SELECT id, collection_date, created FROM specimen WHERE collection_date IN (''1970-01-01'', ''1999-09-09'', ''1899-12-30'', ''1111-01-01'') ORDER BY created DESC.') + WHERE EXISTS ( + SELECT 1 FROM specimen s JOIN __fix_placeholder_dates p ON p.collection_date = s.collection_date + GROUP BY p.collection_date HAVING COUNT(*) > MAX(p.reviewed) + ); +END;--> statement-breakpoint +-- Guard: exhausted Containers (decision 3) +CREATE TRIGGER __fix_exhausted_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot set exhausted Containers to remaining quantity 0: a Container commented exhausted has remaining quantity above 0 and is not one of the 46 the lab reviewed, or its remaining quantity is not 1. Review it first: SELECT id, comment, remaining_quantity FROM storage_container WHERE comment LIKE ''%exhausted%'' AND remaining_quantity > 0.') + WHERE EXISTS ( + SELECT 1 FROM storage_container + WHERE comment LIKE '%exhausted%' AND remaining_quantity > 0 + AND (id NOT IN (SELECT id FROM __fix_exhausted) OR remaining_quantity <> 1) + ); +END;--> statement-breakpoint +-- Guard: placeholder barcodes (decision 4) +CREATE TRIGGER __fix_barcodes_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot empty placeholder barcodes: a micronix tube has an NA-style placeholder barcode (micronix tubes must have one), or more cryovial tubes have one than the 285 the lab reviewed. Review them first.') + WHERE EXISTS ( + SELECT 1 FROM micronix_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' + ) OR ( + SELECT COUNT(*) FROM cryovial_tube WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA' + ) > 285; +END;--> statement-breakpoint +-- Guard: Container 34513 (decision 5) +CREATE TRIGGER __fix_container_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot delete Container 34513: it no longer matches what the lab reviewed (Specimen 31863, no tube, Paper or well row, no derivation, Tag or qPCR well, and its Specimen has another Container). Review it first.') + WHERE EXISTS (SELECT 1 FROM storage_container WHERE id = 34513) + AND NOT ( + (SELECT specimen_id FROM storage_container WHERE id = 34513) = 31863 + AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM cryovial_tube WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM paper WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM static_well WHERE id = 34513) + AND NOT EXISTS (SELECT 1 FROM container_derivation WHERE parent_container_id = 34513 OR child_container_id = 34513) + AND NOT EXISTS (SELECT 1 FROM storage_container_tag WHERE storage_container_id = 34513) + AND NOT EXISTS (SELECT 1 FROM qpcr_experiment_well WHERE storage_container_id = 34513) + AND EXISTS (SELECT 1 FROM storage_container WHERE specimen_id = 31863 AND id <> 34513) + ); +END;--> statement-breakpoint +-- Guard: plate 1760 (decision 8) +CREATE TRIGGER __fix_plate_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot delete plate 1760: it no longer matches what the lab reviewed (named "EXVIVO-27 " with a trailing space, holding no tubes or wells, beside plate 1761 "EXVIVO-27"). Review it first.') + WHERE EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1760) + AND NOT ( + (SELECT name FROM micronix_plate WHERE id = 1760) = 'EXVIVO-27 ' + AND NOT EXISTS (SELECT 1 FROM micronix_tube WHERE collection_id = 1760) + AND NOT EXISTS (SELECT 1 FROM static_well WHERE collection_id = 1760) + AND EXISTS (SELECT 1 FROM micronix_plate WHERE id = 1761 AND name = 'EXVIVO-27') + ); +END;--> statement-breakpoint +-- Guard: Subject pairs to merge (decision 6) +CREATE TRIGGER __fix_merge_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot merge the CTRL Subjects 3d7_0.05 into 3D7_0.05 and 3d7_10k into 3D7_10K: one of them no longer has the name or Study the lab reviewed, or its target Subject is missing. Review them first.') + WHERE EXISTS ( + SELECT 1 FROM __fix_subject_merge m + JOIN study_subject src ON src.id = m.source_id + LEFT JOIN study_subject tgt ON tgt.id = m.target_id + WHERE src.name IS NOT m.source_name + OR tgt.name IS NOT m.target_name + OR tgt.study_id IS NOT src.study_id + OR src.study_id IS NOT (SELECT id FROM study WHERE short_code = 'CTRL') + ); +END;--> statement-breakpoint +-- Guard: no other case-variant Subjects (decision 6) +CREATE TRIGGER __fix_subject_case_tr BEFORE INSERT ON __fix_guard +BEGIN + SELECT RAISE(ABORT, 'Cannot merge the CTRL case-variant Subjects: a Study has Subjects whose names differ only by case beyond the two CTRL pairs the lab reviewed. Review them first. SELECT study_id, group_concat(id || '':'' || name, '', '') FROM study_subject GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 lists them.') + WHERE EXISTS ( + SELECT 1 FROM study_subject + WHERE id NOT IN (SELECT source_id FROM __fix_subject_merge) + GROUP BY study_id, name COLLATE NOCASE HAVING COUNT(*) > 1 + ); +END;--> statement-breakpoint +-- Guard: run every check; aborts the fix on unreviewed rows +INSERT INTO __fix_guard DEFAULT VALUES;--> statement-breakpoint +DROP TRIGGER __fix_dates_tr;--> statement-breakpoint +DROP TRIGGER __fix_exhausted_tr;--> statement-breakpoint +DROP TRIGGER __fix_barcodes_tr;--> statement-breakpoint +DROP TRIGGER __fix_container_tr;--> statement-breakpoint +DROP TRIGGER __fix_plate_tr;--> statement-breakpoint +DROP TRIGGER __fix_merge_tr;--> statement-breakpoint +DROP TRIGGER __fix_subject_case_tr;--> statement-breakpoint +DROP TABLE __fix_guard;--> statement-breakpoint +-- Decision 1: clear placeholder collection dates +UPDATE specimen SET collection_date = NULL +WHERE collection_date IN (SELECT collection_date FROM __fix_placeholder_dates);--> statement-breakpoint +-- Decision 3: exhausted Containers get remaining quantity 0 +UPDATE storage_container SET remaining_quantity = 0 +WHERE id IN (SELECT id FROM __fix_exhausted) AND comment LIKE '%exhausted%' AND remaining_quantity = 1;--> statement-breakpoint +-- Decision 4: empty placeholder cryovial barcodes +UPDATE cryovial_tube SET barcode = NULL +WHERE upper(replace(replace(trim(barcode), '/', ''), '.', '')) = 'NA';--> statement-breakpoint +-- Decision 5: delete Container 34513 +DELETE FROM storage_container WHERE id = 34513;--> statement-breakpoint +-- Decision 8: delete empty plate 1760 +DELETE FROM micronix_plate WHERE id = 1760;--> statement-breakpoint +-- Decision 6: pair each source Specimen with its match +-- As mergeSubjects in src/lib/subjects/merge.ts does it, each source Specimen in turn +-- (lowest id first) is merged into the first target Specimen with the same Specimen type and +-- collection date, moving its Containers there; a source Specimen with no such match moves to +-- the target Subject, where a later source Specimen can match it. Runs after decision 1, so it +-- compares the cleaned dates. +CREATE TEMP TABLE __fix_specimen_merge AS +SELECT s.id AS specimen_id, m.target_id, + COALESCE( + (SELECT MIN(t.id) FROM specimen t + WHERE t.study_subject_id = m.target_id AND t.specimen_type_id = s.specimen_type_id AND t.collection_date IS s.collection_date), + (SELECT MIN(e.id) FROM specimen e + WHERE e.study_subject_id = m.source_id AND e.id < s.id AND e.specimen_type_id = s.specimen_type_id AND e.collection_date IS s.collection_date) + ) AS match_id +FROM specimen s +JOIN __fix_subject_merge m ON m.source_id = s.study_subject_id;--> statement-breakpoint +-- Decision 6: move Containers to the matching target Specimen +UPDATE storage_container +SET specimen_id = (SELECT match_id FROM __fix_specimen_merge WHERE specimen_id = storage_container.specimen_id), + last_updated = datetime('now') +WHERE specimen_id IN (SELECT specimen_id FROM __fix_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +-- Decision 6: move qPCR wells to the matching target Specimen +UPDATE qpcr_experiment_well +SET specimen_id = (SELECT match_id FROM __fix_specimen_merge WHERE specimen_id = qpcr_experiment_well.specimen_id) +WHERE specimen_id IN (SELECT specimen_id FROM __fix_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +-- Decision 6: delete the merged source Specimens +DELETE FROM specimen +WHERE id IN (SELECT specimen_id FROM __fix_specimen_merge WHERE match_id IS NOT NULL);--> statement-breakpoint +-- Decision 6: move unmatched source Specimens to the target Subject +UPDATE specimen +SET study_subject_id = (SELECT target_id FROM __fix_specimen_merge WHERE specimen_id = specimen.id), + last_updated = datetime('now') +WHERE id IN (SELECT specimen_id FROM __fix_specimen_merge WHERE match_id IS NULL);--> statement-breakpoint +-- Decision 6: mark the target Subjects updated +UPDATE study_subject SET last_updated = datetime('now') +WHERE id IN (SELECT target_id FROM __fix_subject_merge);--> statement-breakpoint +-- Decision 6: delete the merged source Subjects +DELETE FROM study_subject WHERE id IN (SELECT source_id FROM __fix_subject_merge);--> statement-breakpoint +DROP TABLE __fix_specimen_merge;--> statement-breakpoint +DROP TABLE __fix_subject_merge;--> statement-breakpoint +DROP TABLE __fix_placeholder_dates;--> statement-breakpoint +DROP TABLE __fix_exhausted; diff --git a/packages/api/package.json b/packages/api/package.json index 5ae1e026..8a9551d4 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -13,7 +13,8 @@ "test:watch": "NODE_ENV=test bun test --watch src eslint-rules", "test:ui": "vitest --ui", "test:coverage": "NODE_ENV=test bun test src eslint-rules --coverage", - "demo:seed": "bun src/lib/demo-seed.ts" + "demo:seed": "bun src/lib/demo-seed.ts", + "data-fix": "bun src/scripts/data-fix.ts" }, "dependencies": { "@sampledb/contract": "workspace:*", diff --git a/packages/api/src/db/README.md b/packages/api/src/db/README.md index 9e17d1b1..53d03d14 100644 --- a/packages/api/src/db/README.md +++ b/packages/api/src/db/README.md @@ -14,6 +14,28 @@ See [ADR 0003: SQLite schema evolution](../../../docs/adr/0003-sqlite-schema-evo 6. Refresh `initial_schema.sql` so **new** empty databases match the latest shape (merge generated SQL; keep `INSERT INTO schema_version` at `CURRENT_SCHEMA_VERSION`). `__tests__/initial-schema-snapshot.test.ts` fails until the snapshot matches a database built by the migrations: columns, types, NOT NULL, defaults, indexes, and foreign keys. 7. Extend the legacy fixtures if the migration touches a table they don't create. Older-version tests in `__tests__/schema-evolution.test.ts` build minimal databases through `createTablesForLaterMigrations`, and `__tests__/client.test.ts` creates its own tables inline; both must contain every table and column a later migration alters. +## One-time data fixes + +A migration ships to every install and runs at startup, so it must be correct on any database: it may not name rows by id or carry out one lab's decisions about its own data, and it must not abort on data another install could legitimately hold. A change to one install's data is a **data fix** (CONTEXT.md) instead: a SQL file an operator runs by hand. + +```bash +bun run data-fix [--db=path/to/database.sqlite] [--dry-run] +``` + +`src/scripts/data-fix.ts` is the CLI; `runDataFix` in `data-fix.ts` does the work. It opens the file directly, not through `openOperationalDatabase`, because a data fix must not migrate the database, and it: + +1. refuses a database whose `schema_version` is not `CURRENT_SCHEMA_VERSION`; +2. does nothing, and says so, if `data_fix_log` already records the fix's name with the same checksum, and refuses if the checksum differs; +3. copies the database with `VACUUM INTO` (as `migration-backup.ts` does before migrations) to `.pre-data-fix--.bak`; +4. runs the statements (split on `--> statement-breakpoint`) in one `BEGIN IMMEDIATE` transaction, with foreign keys on, and records `(name, checksum, applied_at)` in `data_fix_log` in the same transaction; +5. prints the rows each statement changed (`total_changes()`, so trigger and foreign-key changes count), labelled by the `--` comment just above it. + +`--dry-run` does steps 1, 2 and 4 in a transaction it rolls back, and writes no copy and no record. + +`data_fix_log` belongs to the runner, not the operational schema: it is created on the first fix and is absent from `schema.ts`, `initial_schema.sql`, and databases no fix ran on. + +**Fix files are site-specific.** `ops/data-fixes/` holds the EPPIcenter lab's fixes for its own database; other installs must not run them. Start each file with a header naming the database it is for, the decision it carries out, and a warning not to run it elsewhere. Guard it: check that the rows it changes are the ones that were reviewed, and abort (for example with a temp trigger that `RAISE(ABORT, …)`s, as `2026-11-01-eppicenter-legacy-cleanup.sql` does) when they are not. A test under `__tests__/data-fixes/` runs the lab's fix against a database seeded in production's shape. The operator steps are in the deployment guide in `packages/docs` (One-time data fixes). + ## How evolution works | Database state | Behaviour | diff --git a/packages/api/src/db/__tests__/data-fix.test.ts b/packages/api/src/db/__tests__/data-fix.test.ts new file mode 100644 index 00000000..e14ce2a5 --- /dev/null +++ b/packages/api/src/db/__tests__/data-fix.test.ts @@ -0,0 +1,127 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Database } from 'bun:sqlite' +import { mkdtempSync, readdirSync, rmSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { applyInitialSchema } from '../apply-initial-schema' +import { CURRENT_SCHEMA_VERSION } from '../schema-evolution' +import { DataFixError, runDataFix, type DataFix } from '../data-fix' + +const NOW = '2024-01-01 00:00:00' + +const renameStudies: DataFix = { + name: '2026-11-01-rename-studies', + sql: [ + '-- Header comment', + "-- Rename the pilot study\nUPDATE study SET title = 'Pilot (renamed)' WHERE short_code = 'PIL'", + "UPDATE study SET lead_person = 'Lab' WHERE short_code IN ('PIL', 'MAIN')", + "CREATE TEMP TABLE __fix_scratch (id INTEGER);\n", + 'DROP TABLE __fix_scratch', + ].join(';--> statement-breakpoint\n'), +} + +function titles(sqlite: Database): string[] { + return (sqlite.prepare('SELECT title FROM study ORDER BY id').all() as Array<{ title: string }>).map((r) => r.title) +} + +describe('runDataFix', () => { + let dir: string + let sqlite: Database + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'data-fix-')) + sqlite = new Database(join(dir, 'lab.sqlite')) + sqlite.exec('PRAGMA foreign_keys = ON') + applyInitialSchema(sqlite) + sqlite.exec(`INSERT INTO study (id, title, short_code, is_longitudinal, lead_person, created, last_updated) VALUES + (1, 'Pilot', 'PIL', 0, 'A', '${NOW}', '${NOW}'), (2, 'Main', 'MAIN', 0, 'B', '${NOW}', '${NOW}')`) + }) + + afterEach(() => { + sqlite.close() + rmSync(dir, { recursive: true, force: true }) + }) + + it('applies the fix, reports the rows each statement changed, and records it', () => { + const result = runDataFix(sqlite, renameStudies, { now: new Date('2026-11-01T12:00:00Z') }) + + expect(result.status).toBe('applied') + expect(result.statements).toEqual([ + { label: 'Rename the pilot study', changes: 1 }, + { label: "UPDATE study SET lead_person = 'Lab' WHERE short_code IN ('PIL', 'MAIN')", changes: 2 }, + { label: 'CREATE TEMP TABLE __fix_scratch (id INTEGER)', changes: 0 }, + { label: 'DROP TABLE __fix_scratch', changes: 0 }, + ]) + expect(titles(sqlite)).toEqual(['Pilot (renamed)', 'Main']) + expect(sqlite.prepare('SELECT name, applied_at, checksum FROM data_fix_log').all()).toEqual([ + { name: '2026-11-01-rename-studies', applied_at: '2026-11-01T12:00:00.000Z', checksum: expect.stringMatching(/^[0-9a-f]{64}$/) }, + ]) + }) + + it('does nothing the second time and says when it ran', () => { + runDataFix(sqlite, renameStudies, { now: new Date('2026-11-01T12:00:00Z') }) + sqlite.exec(`UPDATE study SET title = 'Pilot' WHERE id = 1`) + + const again = runDataFix(sqlite, renameStudies) + + expect(again).toEqual({ status: 'already-applied', name: renameStudies.name, appliedAt: '2026-11-01T12:00:00.000Z' }) + expect(titles(sqlite)).toEqual(['Pilot', 'Main']) + }) + + it('refuses a fix whose file changed since it ran under the same name', () => { + runDataFix(sqlite, renameStudies) + + expect(() => runDataFix(sqlite, { ...renameStudies, sql: `${renameStudies.sql};\nSELECT 1` })).toThrow( + /2026-11-01-rename-studies already ran .* with different contents/, + ) + }) + + it('copies the database before changing it, naming the copy for the fix', () => { + const result = runDataFix(sqlite, renameStudies, { now: new Date('2026-11-01T12:00:00Z') }) + + expect(result.status === 'applied' && result.backupPath).toBe( + join(dir, 'lab.sqlite.pre-data-fix-2026-11-01-rename-studies-20261101T120000000Z.bak'), + ) + const copy = new Database(join(dir, 'lab.sqlite.pre-data-fix-2026-11-01-rename-studies-20261101T120000000Z.bak'), { readonly: true }) + expect(titles(copy)).toEqual(['Pilot', 'Main']) + copy.close() + }) + + it('in a dry run reports the counts, rolls everything back and records nothing', () => { + const result = runDataFix(sqlite, renameStudies, { dryRun: true }) + + expect(result.status).toBe('dry-run') + expect(result.status === 'dry-run' && result.statements.map((s) => s.changes)).toEqual([1, 2, 0, 0]) + expect(titles(sqlite)).toEqual(['Pilot', 'Main']) + expect(sqlite.prepare(`SELECT name FROM sqlite_master WHERE name = 'data_fix_log'`).all()).toEqual([]) + expect(readdirSync(dir).filter((f) => f.endsWith('.bak'))).toEqual([]) + }) + + it('in a dry run of a fix that already ran says so', () => { + runDataFix(sqlite, renameStudies, { now: new Date('2026-11-01T12:00:00Z') }) + + expect(runDataFix(sqlite, renameStudies, { dryRun: true }).status).toBe('already-applied') + }) + + it('refuses a database that is not at the current schema version, changing nothing', () => { + sqlite.exec(`UPDATE schema_version SET version = ${CURRENT_SCHEMA_VERSION - 1}`) + + expect(() => runDataFix(sqlite, renameStudies)).toThrow( + new RegExp(`schema version ${CURRENT_SCHEMA_VERSION - 1}.*needs ${CURRENT_SCHEMA_VERSION}`), + ) + expect(titles(sqlite)).toEqual(['Pilot', 'Main']) + expect(readdirSync(dir).filter((f) => f.endsWith('.bak'))).toEqual([]) + }) + + it('rolls back every statement when one fails, names it, and records nothing', () => { + const failing: DataFix = { + name: 'failing', + sql: `UPDATE study SET title = title || ' (changed)';--> statement-breakpoint\nUPDATE study SET short_code = 'MAIN' WHERE id = 1`, + } + + expect(() => runDataFix(sqlite, failing)).toThrow(DataFixError) + expect(() => runDataFix(sqlite, failing)).toThrow(/statement 2 .*UNIQUE constraint failed/) + expect(titles(sqlite)).toEqual(['Pilot', 'Main']) + expect(sqlite.prepare(`SELECT name FROM sqlite_master WHERE name = 'data_fix_log'`).all()).toEqual([]) + }) +}) diff --git a/packages/api/src/db/__tests__/data-fixes/eppicenter-legacy-cleanup.test.ts b/packages/api/src/db/__tests__/data-fixes/eppicenter-legacy-cleanup.test.ts new file mode 100644 index 00000000..b09380e1 --- /dev/null +++ b/packages/api/src/db/__tests__/data-fixes/eppicenter-legacy-cleanup.test.ts @@ -0,0 +1,305 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest' +import { Database } from 'bun:sqlite' +import { resolve } from 'path' +import { applyInitialSchema } from '../../apply-initial-schema' +import { loadDataFix, runDataFix } from '../../data-fix' +import { getMonorepoRoot } from '../../open' + +/** + * The EPPIcenter lab's one-time legacy data cleanup (issues #124 and #126), a site-specific data + * fix in ops/data-fixes that only the lab runs. Each test builds a database in production's + * shape: the current snapshot, without the index production lacks, seeded with the production + * ids each decision names. + */ + +const FIX = loadDataFix(resolve(getMonorepoRoot(), 'ops/data-fixes/2026-11-01-eppicenter-legacy-cleanup.sql')) + +function applyFix(sqlite: Database): void { + runDataFix(sqlite, FIX) +} + +const NOW = '2024-01-01 00:00:00' + +function productionShapedDatabase(): Database { + const sqlite = new Database(':memory:') + sqlite.exec('PRAGMA foreign_keys=ON') + applyInitialSchema(sqlite) + // Production has no Specimen identity index (decision 7). + sqlite.exec('DROP INDEX IF EXISTS idx_specimen_study_subject_type_date') + sqlite.exec(`INSERT INTO study (id, title, short_code, is_longitudinal, lead_person, created, last_updated) VALUES + (3, 'Controls', 'CTRL', 0, 'Lab', '${NOW}', '${NOW}'), (8, 'Jiangsu', 'JIANGSU17', 0, 'Lab', '${NOW}', '${NOW}')`) + sqlite.exec(`INSERT INTO specimen_type (id, name, created, last_updated) VALUES (2, 'DBS', '${NOW}', '${NOW}'), (21, 'WB', '${NOW}', '${NOW}')`) + sqlite.exec(`INSERT INTO unit (id, symbol, name, category) VALUES (5, 'uL', 'microliter', 'volume')`) + sqlite.exec(`INSERT INTO storage_type (id, name) VALUES (1, 'Freezer')`) + sqlite.exec(`INSERT INTO location (id, name, storage_type_id) VALUES (107, 'Freezer A', 1)`) + sqlite.exec(`INSERT INTO cryovial_box (id, location_id, name, created, last_updated) VALUES (900, 107, 'Box 900', '${NOW}', '${NOW}')`) + return sqlite +} + +function addSubject(sqlite: Database, id: number, studyId: number, name: string): void { + sqlite.prepare(`INSERT INTO study_subject (id, study_id, name, created, last_updated) VALUES (?, ?, ?, ?, ?)`).run(id, studyId, name, NOW, NOW) +} + +function addSpecimen(sqlite: Database, id: number, subjectId: number, typeId: number, date: string | null): void { + sqlite + .prepare(`INSERT INTO specimen (id, study_subject_id, specimen_type_id, collection_date, created, last_updated) VALUES (?, ?, ?, ?, ?, ?)`) + .run(id, subjectId, typeId, date, NOW, NOW) +} + +function addContainer(sqlite: Database, id: number, specimenId: number, options: { comment?: string; remaining?: number } = {}): void { + sqlite + .prepare(`INSERT INTO storage_container (id, specimen_id, comment, total_quantity, remaining_quantity, unit_id, created, last_updated) VALUES (?, ?, ?, 1.0, ?, 5, ?, ?)`) + .run(id, specimenId, options.comment ?? null, options.remaining ?? 1.0, NOW, NOW) +} + +function addCryovialTube(sqlite: Database, id: number, barcode: string | null): void { + sqlite.prepare(`INSERT INTO cryovial_tube (id, collection_id, barcode) VALUES (?, 900, ?)`).run(id, barcode) +} + +function addMicronixTube(sqlite: Database, id: number, plateId: number, barcode: string): void { + sqlite.prepare(`INSERT INTO micronix_tube (id, collection_id, barcode) VALUES (?, ?, ?)`).run(id, plateId, barcode) +} + +/** Production's rows for every decision, plus neighbours each step must leave alone. */ +function seedProductionRows(sqlite: Database): void { + addSubject(sqlite, 31047, 3, '3d7_0.05') + addSubject(sqlite, 66537, 3, '3D7_0.05') + addSubject(sqlite, 31049, 3, '3d7_10k') + addSubject(sqlite, 66543, 3, '3D7_10K') + addSubject(sqlite, 25613, 8, 'JS087') + addSubject(sqlite, 500, 8, 'S500') + + // Decision 6: Specimen 43644 has no match on the target and moves to it; Specimen 43645 + // matches target Specimen 120524 (same type and date) and is merged into it. + addSpecimen(sqlite, 43644, 31047, 21, '2018-03-15') + addSpecimen(sqlite, 43645, 31047, 2, '2021-06-07') + addSpecimen(sqlite, 120523, 66537, 2, '2021-06-08') + addSpecimen(sqlite, 120524, 66537, 2, '2021-06-07') + addSpecimen(sqlite, 43646, 31049, 21, '2018-03-15') + addSpecimen(sqlite, 120529, 66543, 2, '2021-06-07') + addContainer(sqlite, 70001, 43644) + addContainer(sqlite, 70002, 43645) + addContainer(sqlite, 70003, 43645) + addContainer(sqlite, 70004, 120524) + addContainer(sqlite, 70005, 43646) + + // Decision 1: placeholder dates, and dates that stay (including a future one, decision 2). + addSpecimen(sqlite, 1, 500, 2, '1970-01-01') + addSpecimen(sqlite, 2, 500, 2, '1999-09-09') + addSpecimen(sqlite, 3, 500, 2, '1899-12-30') + addSpecimen(sqlite, 4, 500, 2, '1111-01-01') + addSpecimen(sqlite, 5, 500, 2, '1970-01-02') + addSpecimen(sqlite, 6, 500, 2, '2073-02-08') + addSpecimen(sqlite, 7, 500, 21, '1970-01-01') + + // Decision 3: exhausted Containers with remaining 1, and ones that stay. + addSpecimen(sqlite, 10, 500, 21, '2020-01-01') + addContainer(sqlite, 50465, 10, { comment: 'exhausted' }) + addContainer(sqlite, 124552, 10, { comment: 'Z32592; exhausted' }) + addContainer(sqlite, 143335, 10, { comment: 'Z20759; Zymo cleaned; exhausted' }) + addContainer(sqlite, 74034, 10, { comment: 'Reuploaded & Set to Exhausted', remaining: 0 }) + addContainer(sqlite, 74035, 10, { comment: 'half used', remaining: 0.5 }) + + // Decision 4: placeholder tube barcodes, and real barcodes that contain NA. + for (const [id, barcode] of [ + [80001, 'NA'], + [80002, 'N/A'], + [80003, 'n/a'], + [80004, ' NA '], + [80005, 'NA98'], + [80006, 'DFNA'], + [80007, null], + ] as const) { + addContainer(sqlite, id, 10) + addCryovialTube(sqlite, id, barcode) + } + + // Decision 5: Container 34513 has no tube or Paper row; its Specimen keeps its other Container. + addSpecimen(sqlite, 31863, 25613, 21, '2018-07-24') + addContainer(sqlite, 34513, 31863) + addContainer(sqlite, 34514, 31863) + addCryovialTube(sqlite, 34514, 'JS087-1') + + // Decision 8: plate 1760 is empty; plate 1761 keeps its tubes. + sqlite.exec(`INSERT INTO micronix_plate (id, location_id, name, created, last_updated) VALUES + (1760, 107, 'EXVIVO-27 ', '${NOW}', '${NOW}'), (1761, 107, 'EXVIVO-27', '${NOW}', '${NOW}')`) + addContainer(sqlite, 90001, 10) + addMicronixTube(sqlite, 90001, 1761, '1000000001') +} + +function column(sqlite: Database, sql: string): T[] { + return (sqlite.prepare(sql).all() as Array>).map((row) => Object.values(row)[0] as T) +} + +function rows(sqlite: Database, sql: string): unknown[] { + return sqlite.prepare(sql).all() +} + +describe('data fix 2026-11-01-eppicenter-legacy-cleanup', () => { + let sqlite: Database + + beforeEach(() => { + sqlite = productionShapedDatabase() + }) + + afterEach(() => { + sqlite.close() + }) + + it('applies each decision to the rows it names and leaves every other row alone', () => { + seedProductionRows(sqlite) + + applyFix(sqlite) + + // Decision 1 + expect(rows(sqlite, 'SELECT id, collection_date FROM specimen WHERE id <= 7 ORDER BY id')).toEqual([ + { id: 1, collection_date: null }, + { id: 2, collection_date: null }, + { id: 3, collection_date: null }, + { id: 4, collection_date: null }, + { id: 5, collection_date: '1970-01-02' }, + { id: 6, collection_date: '2073-02-08' }, + { id: 7, collection_date: null }, + ]) + // Decision 3 + expect(rows(sqlite, 'SELECT id, comment, remaining_quantity FROM storage_container WHERE specimen_id = 10 AND comment IS NOT NULL ORDER BY id')).toEqual([ + { id: 50465, comment: 'exhausted', remaining_quantity: 0 }, + { id: 74034, comment: 'Reuploaded & Set to Exhausted', remaining_quantity: 0 }, + { id: 74035, comment: 'half used', remaining_quantity: 0.5 }, + { id: 124552, comment: 'Z32592; exhausted', remaining_quantity: 0 }, + { id: 143335, comment: 'Z20759; Zymo cleaned; exhausted', remaining_quantity: 0 }, + ]) + // Decision 4 + expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube WHERE id BETWEEN 80001 AND 80007 ORDER BY id')).toEqual([ + { id: 80001, barcode: null }, + { id: 80002, barcode: null }, + { id: 80003, barcode: null }, + { id: 80004, barcode: null }, + { id: 80005, barcode: 'NA98' }, + { id: 80006, barcode: 'DFNA' }, + { id: 80007, barcode: null }, + ]) + // Decision 5 + expect(column(sqlite, 'SELECT id FROM storage_container WHERE specimen_id = 31863')).toEqual([34514]) + // Decision 8 + expect(column(sqlite, `SELECT id FROM micronix_plate WHERE trim(name) = 'EXVIVO-27'`)).toEqual([1761]) + expect(column(sqlite, 'SELECT COUNT(*) FROM micronix_tube WHERE collection_id = 1761')).toEqual([1]) + // Decision 6: the app's Subject merge, run for both pairs. + expect(rows(sqlite, 'SELECT id, name FROM study_subject WHERE study_id = 3 ORDER BY id')).toEqual([ + { id: 66537, name: '3D7_0.05' }, + { id: 66543, name: '3D7_10K' }, + ]) + expect(rows(sqlite, 'SELECT id, study_subject_id FROM specimen WHERE id IN (43644, 43645, 43646, 120523, 120524, 120529) ORDER BY id')).toEqual([ + { id: 43644, study_subject_id: 66537 }, + { id: 43646, study_subject_id: 66543 }, + { id: 120523, study_subject_id: 66537 }, + { id: 120524, study_subject_id: 66537 }, + { id: 120529, study_subject_id: 66543 }, + ]) + expect(rows(sqlite, 'SELECT id, specimen_id FROM storage_container WHERE id BETWEEN 70001 AND 70005 ORDER BY id')).toEqual([ + { id: 70001, specimen_id: 43644 }, + { id: 70002, specimen_id: 120524 }, + { id: 70003, specimen_id: 120524 }, + { id: 70004, specimen_id: 120524 }, + { id: 70005, specimen_id: 43646 }, + ]) + expect(column(sqlite, 'SELECT last_updated FROM study_subject WHERE id IN (66537, 66543)').every((t) => t !== NOW)).toBe(true) + expect(column(sqlite, 'SELECT last_updated FROM specimen WHERE id = 43644')).not.toEqual([NOW]) + expect(column(sqlite, 'SELECT last_updated FROM storage_container WHERE id = 70002')).not.toEqual([NOW]) + expect(column(sqlite, `SELECT name FROM sqlite_temp_master WHERE name LIKE '\\_\\_fix\\_%' ESCAPE '\\'`)).toEqual([]) + expect(column(sqlite, 'PRAGMA foreign_key_check')).toEqual([]) + }) + + it('changes nothing on a database without the legacy rows', () => { + addSubject(sqlite, 1, 3, 'A') + addSpecimen(sqlite, 1, 1, 2, '2020-01-01') + addContainer(sqlite, 1, 1, { comment: 'exhausted', remaining: 0 }) + addCryovialTube(sqlite, 1, 'CV1') + + applyFix(sqlite) + + expect(rows(sqlite, 'SELECT id, study_subject_id, collection_date FROM specimen')).toEqual([{ id: 1, study_subject_id: 1, collection_date: '2020-01-01' }]) + expect(rows(sqlite, 'SELECT id, remaining_quantity FROM storage_container')).toEqual([{ id: 1, remaining_quantity: 0 }]) + expect(rows(sqlite, 'SELECT id, barcode FROM cryovial_tube')).toEqual([{ id: 1, barcode: 'CV1' }]) + expect(column(sqlite, 'SELECT COUNT(*) FROM study_subject')).toEqual([1]) + }) + + const aborts: Array<[string, (sqlite: Database) => void, RegExp]> = [ + [ + 'an exhausted Container the lab did not review still has quantity', + (db) => addContainer(db, 99999, 10, { comment: 'exhausted' }), + /exhausted/, + ], + [ + 'a reviewed exhausted Container has a remaining quantity other than 1', + (db) => db.exec('UPDATE storage_container SET remaining_quantity = 0.5 WHERE id = 50465'), + /exhausted/, + ], + [ + 'there are more Specimens on a placeholder date than the lab reviewed', + (db) => addSpecimen(db, 99999, 500, 21, '1111-01-01'), + /placeholder collection date/, + ], + [ + 'a micronix tube has a placeholder barcode, which cannot be emptied', + (db) => { + addContainer(db, 99999, 10) + addMicronixTube(db, 99999, 1761, 'N/A') + }, + /placeholder barcode/, + ], + [ + 'Container 34513 has a tube row', + (db) => addCryovialTube(db, 34513, 'X34513'), + /Container 34513/, + ], + [ + 'Container 34513 is the only Container of its Specimen', + (db) => db.exec('DELETE FROM cryovial_tube WHERE id = 34514; DELETE FROM storage_container WHERE id = 34514'), + /Container 34513/, + ], + [ + 'plate 1760 holds a tube', + (db) => { + addContainer(db, 99999, 10) + addMicronixTube(db, 99999, 1760, '1000000002') + }, + /plate 1760/, + ], + [ + 'plate 1760 has been renamed', + (db) => db.exec(`UPDATE micronix_plate SET name = 'EXVIVO-28' WHERE id = 1760`), + /plate 1760/, + ], + [ + 'a Subject to merge has been renamed', + (db) => db.exec(`UPDATE study_subject SET name = '3d7_10k_b' WHERE id = 31049`), + /merge/, + ], + [ + 'a Subject to merge has lost its target', + (db) => db.exec(`UPDATE specimen SET study_subject_id = 31049 WHERE study_subject_id = 66543; DELETE FROM study_subject WHERE id = 66543`), + /merge/, + ], + [ + 'another Study has Subjects whose names differ only by case', + (db) => { + addSubject(db, 600, 8, 'abc') + addSubject(db, 601, 8, 'ABC') + }, + /differ only by case/, + ], + ] + + for (const [situation, change, message] of aborts) { + it(`aborts, changing nothing, when ${situation}`, () => { + seedProductionRows(sqlite) + change(sqlite) + const before = sqlite.serialize() + + expect(() => applyFix(sqlite)).toThrow(message) + + expect(Buffer.from(sqlite.serialize()).equals(Buffer.from(before))).toBe(true) + }) + } +}) diff --git a/packages/api/src/db/data-fix.ts b/packages/api/src/db/data-fix.ts new file mode 100644 index 00000000..4263f597 --- /dev/null +++ b/packages/api/src/db/data-fix.ts @@ -0,0 +1,180 @@ +import type { Database } from 'bun:sqlite' +import { createHash } from 'crypto' +import { readFileSync } from 'fs' +import { basename } from 'path' +import { databaseCopyPath, writeDatabaseCopy } from './migration-backup' +import { splitSqlStatements } from './migration-runner' +import { CURRENT_SCHEMA_VERSION, getRecordedSchemaVersion } from './schema-evolution' + +/** + * A one-time data fix: SQL one site runs by hand against its own database (README, "One-time + * data fixes"). Unlike a numbered migration it ships to no other install, so it may name that + * site's rows. Statements are separated with `--> statement-breakpoint`, as in migrations. + */ +export interface DataFix { + /** Recorded in `data_fix_log`; the file name without `.sql`. */ + name: string + sql: string +} + +export interface DataFixStatementResult { + /** The first line of the comment block just above the statement, or the statement's first line. */ + label: string + /** Rows inserted, updated or deleted, including by triggers and foreign key actions. */ + changes: number +} + +export type DataFixResult = + | { status: 'applied'; name: string; backupPath: string | null; statements: DataFixStatementResult[] } + | { status: 'dry-run'; name: string; statements: DataFixStatementResult[] } + | { status: 'already-applied'; name: string; appliedAt: string } + +export interface RunDataFixOptions { + /** Run the fix in a transaction that is rolled back, and report what it would change. */ + dryRun?: boolean + now?: Date +} + +export class DataFixError extends Error { + constructor(message: string, cause?: unknown) { + super(message) + this.name = 'DataFixError' + if (cause instanceof Error) this.cause = cause + } +} + +const LOG_TABLE = 'data_fix_log' + +export function loadDataFix(path: string): DataFix { + return { name: basename(path).replace(/\.sql$/i, ''), sql: readFileSync(path, 'utf-8') } +} + +function checksum(sql: string): string { + return createHash('sha256').update(sql).digest('hex') +} + +interface Statement { + sql: string + label: string +} + +/** The fix's statements; a chunk holding only comments (such as the file header) is skipped. */ +function statementsOf(sql: string): Statement[] { + const statements: Statement[] = [] + for (const chunk of splitSqlStatements(sql)) { + const lines = chunk.split('\n').map((line) => line.trim()) + const firstSql = lines.findIndex((line) => line !== '' && !line.startsWith('--')) + if (firstSql === -1 || lines.slice(firstSql).join('').replace(/;/g, '').trim() === '') continue + let commentStart = firstSql + while (commentStart > 0 && lines[commentStart - 1]!.startsWith('--')) commentStart-- + const comment = commentStart < firstSql ? lines[commentStart]!.replace(/^--\s*/, '') : undefined + const label = comment || lines[firstSql]!.replace(/;$/, '') + statements.push({ sql: chunk, label: label.length > 120 ? `${label.slice(0, 117)}...` : label }) + } + return statements +} + +function loggedRun(sqlite: Database, name: string): { checksum: string; applied_at: string } | null { + const table = sqlite.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(LOG_TABLE) + if (!table) return null + return sqlite.prepare(`SELECT checksum, applied_at FROM ${LOG_TABLE} WHERE name = ?`).get(name) as { + checksum: string + applied_at: string + } | null +} + +function totalChanges(sqlite: Database): number { + return (sqlite.prepare('SELECT total_changes() AS n').get() as { n: number }).n +} + +/** Run every statement inside an open transaction, counting the rows each one changed. */ +function runStatements(sqlite: Database, fix: DataFix): DataFixStatementResult[] { + const results: DataFixStatementResult[] = [] + statementsOf(fix.sql).forEach((statement, index) => { + const before = totalChanges(sqlite) + try { + sqlite.exec(statement.sql) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + throw new DataFixError( + `Data fix ${fix.name} failed at statement ${index + 1} (${statement.label}): ${detail}. ` + + 'Every statement was rolled back; the database is unchanged.', + error, + ) + } + results.push({ label: statement.label, changes: totalChanges(sqlite) - before }) + }) + return results +} + +/** + * Run a one-time data fix against a database at CURRENT_SCHEMA_VERSION, in one write + * transaction, and record it in `data_fix_log` so a second run does nothing. A real run first + * copies the database next to its file. Throws DataFixError, with the database unchanged, when + * the schema version differs, the same name ran with different contents, the copy fails, or a + * statement fails. + */ +export function runDataFix(sqlite: Database, fix: DataFix, options: RunDataFixOptions = {}): DataFixResult { + const now = options.now ?? new Date() + const version = getRecordedSchemaVersion(sqlite) + if (version !== CURRENT_SCHEMA_VERSION) { + throw new DataFixError( + `The database is at schema version ${version ?? 'none'}, and data fix ${fix.name} needs ${CURRENT_SCHEMA_VERSION}. ` + + 'Start this version of SampleDB on the database once so it migrates, stop it, then run the fix.', + ) + } + + const sum = checksum(fix.sql) + const previous = loggedRun(sqlite, fix.name) + if (previous) { + if (previous.checksum !== sum) { + throw new DataFixError( + `Data fix ${fix.name} already ran on ${previous.applied_at} with different contents ` + + `(checksum ${previous.checksum}, the file now has ${sum}). Nothing was run. ` + + 'Put further changes in a new data fix with its own name.', + ) + } + return { status: 'already-applied', name: fix.name, appliedAt: previous.applied_at } + } + + if (options.dryRun) { + sqlite.exec('BEGIN IMMEDIATE') + try { + return { status: 'dry-run', name: fix.name, statements: runStatements(sqlite, fix) } + } finally { + sqlite.exec('ROLLBACK') + } + } + + const backupPath = databaseCopyPath(sqlite, `pre-data-fix-${fix.name}`, now) + if (backupPath != null) { + try { + writeDatabaseCopy(sqlite, backupPath) + } catch (error: unknown) { + const detail = error instanceof Error ? error.message : String(error) + throw new DataFixError( + `Could not copy ${sqlite.filename} to ${backupPath} before data fix ${fix.name}: ${detail}. ` + + 'Nothing was run. Make the directory writable (and check free space), then run it again.', + error, + ) + } + } + + sqlite.exec('BEGIN IMMEDIATE') + try { + const statements = runStatements(sqlite, fix) + // The runner's own record, not part of the operational schema: only databases a fix ran on + // have it. Created in the fix's transaction, so a failed first fix leaves no trace. + sqlite.exec(`CREATE TABLE IF NOT EXISTS ${LOG_TABLE} ( + name TEXT PRIMARY KEY NOT NULL, + checksum TEXT NOT NULL, + applied_at TEXT NOT NULL + )`) + sqlite.prepare(`INSERT INTO ${LOG_TABLE} (name, checksum, applied_at) VALUES (?, ?, ?)`).run(fix.name, sum, now.toISOString()) + sqlite.exec('COMMIT') + return { status: 'applied', name: fix.name, backupPath, statements } + } catch (error: unknown) { + sqlite.exec('ROLLBACK') + throw error + } +} diff --git a/packages/api/src/db/migration-backup.ts b/packages/api/src/db/migration-backup.ts index e98f5b51..4075e9cb 100644 --- a/packages/api/src/db/migration-backup.ts +++ b/packages/api/src/db/migration-backup.ts @@ -26,23 +26,42 @@ function backupTimestamp(now: Date): string { } /** - * Write a consistent copy of the database next to its file before migrations run, so a - * migration that goes wrong can be undone without falling back to the last restic snapshot. - * `VACUUM INTO` reads through the connection, so the copy includes pages still in the WAL. + * Where to copy an on-disk database before changing it: next to its file, named + * `.